C++ readToken函数代码示例

bool Reader::readArray(Token& tokenStart) {
  currentValue() = Value(arrayValue);
  currentValue().setOffsetStart(tokenStart.start_ - begin_);
  skipSpaces();
  if (*current_ == ']') // empty array
  {
    Token endArray;
    readToken(endArray);
    return true;
  }
  int index = 0;
  for (;;) {
    Value& value = currentValue()[index++];
    nodes_.push(&value);
    bool ok = readValue();
    nodes_.pop();
    if (!ok) // error already set
      return recoverFromError(tokenArrayEnd);

    Token token;
    // Accept Comment after last item in the array.
    ok = readToken(token);
    while (token.type_ == tokenComment && ok) {
      ok = readToken(token);
    }
    bool badTokenType =
        (token.type_ != tokenArraySeparator && token.type_ != tokenArrayEnd);
    if (!ok || badTokenType) {
      return addErrorAndRecover(
          "Missing ',' or ']' in array declaration", token, tokenArrayEnd);
    }
    if (token.type_ == tokenArrayEnd)
      break;
  }
  return true;
}

/* parses a trait:
 * 	trait Foo {} */
static boolean parseTrait (tokenInfo *const token)
{
	boolean readNext = TRUE;
	tokenInfo *name;

	readToken (token);
	if (token->type != TOKEN_IDENTIFIER)
		return FALSE;

	name = newToken ();
	copyToken (name, token, TRUE);

	makeSimplePhpTag (name, K_TRAIT, ACCESS_UNDEFINED);

	readToken (token);
	if (token->type == TOKEN_OPEN_CURLY)
		enterScope (token, name->string, K_TRAIT);
	else
		readNext = FALSE;

	deleteToken (name);

	return readNext;
}

inline bool Json::next(const JsonValue& _current, JsonValue& _next)
{
	m_readPos = m_fileContent.data() + _current.offset;
	// Iterate through the stream to the next ','. Meanwhile the number of
	// brackets ({} and []) must match.
	int parenthesis = 0;
	do {
		if(!readToken()) return false;
		if(m_tokenPos[0] == '{' || m_tokenPos[0] == '[') ++parenthesis;
		else if(m_tokenPos[0] == '}' || m_tokenPos[0] == ']') --parenthesis;
	} while(parenthesis > 0 || m_tokenPos[0] != ',');
	if(parenthesis < 0) return false;
	readProperty(_next);
	return true;
}

static void parseClass (tokenInfo *const token)
{
	Assert (isKeyword (token, KEYWORD_class));
	readToken (token);
	if (isType (token, TOKEN_IDENTIFIER))
	{
#ifndef TYPE_REFERENCE_TOOL
		makeEiffelClassTag (token);
		readToken (token);
#else
		vStringCopy (token->className, token->string);
		vStringUpper (token->className);
		if (PrintClass)
			puts (vStringValue (token->className));
		if (! PrintReferences)
			exit (0);
		readToken (token);
#endif
	}

	do
	{
		if (isType (token, TOKEN_OPEN_BRACKET))
			parseGeneric (token, TRUE);
		else if (! isType (token, TOKEN_KEYWORD))
			readToken (token);
		else switch (token->keyword)
		{
			case KEYWORD_inherit:  parseInherit (token);        break;
			case KEYWORD_feature:  parseFeatureClauses (token); break;
			case KEYWORD_convert:  parseConvert (token);        break;
			default:               readToken (token);           break;
		}
	} while (! isKeyword (token, KEYWORD_end) &&
	         ! isType (token, TOKEN_EOF));
}

static void parseRename (tokenInfo *const token)
{
	do {
		readToken (token);
		if (readFeatureName (token))
		{
			readToken (token);
			if (isKeyword (token, KEYWORD_as))
			{
				readToken (token);
				if (readFeatureName (token))
				{
#ifndef TYPE_REFERENCE_TOOL
					makeEiffelFeatureTag (token);  /* renamed feature */
#endif
					readToken (token);
				}
			}
		}
	} while (isType (token, TOKEN_COMMA));

	findKeyword (token, KEYWORD_end);
	readToken (token);
}

PARSENODE_PTR SQLParser::parseInsert() {
    if (!startsInsert(nowReading)) {
        syntaxError(nowReading, "expect insert token!");        
        return nullptr;
    }
    //LOG_TRACE(logger, "parse insert statement.");

    PARSENODE_PTR insertNode = PARSENODE_PTR(new ParseNode(INSERT));
    readToken();
    expect(INTO);
    insertNode->children.push_back(parseIdentifier());
    expect(VALUES);

    expect(LEFT_BRACE);
    insertNode->children.push_back(parseLiteral());
    while (nowReading == SLICE) {
        readToken();
        insertNode->children.push_back(parseLiteral());
    }
    expect(RIGHT_BRACE);

    expect(TERMINATOR);
    return insertNode;
}

static void parseConvert (tokenInfo *const token)
{
	Assert (isKeyword (token, KEYWORD_convert));
	do
	{
		readToken (token);
		if (! isType (token, TOKEN_IDENTIFIER))
			break;
		else if (isType (token, TOKEN_OPEN_PAREN))
		{
			while (! isType (token, TOKEN_CLOSE_PAREN) &&
			       ! isType (token, TOKEN_EOF))
				readToken (token);
		}
		else if (isType (token, TOKEN_COLON))
		{
			readToken (token);
			if (! isType (token, TOKEN_OPEN_BRACE))
				break;
			else while (! isType (token, TOKEN_CLOSE_BRACE))
				readToken (token);
		}
	} while (isType (token, TOKEN_COMMA));
}

bool DbcParser::expectData(DbcParser::DbcTokenList &tokens, dbc_token_type_t type, QString *data, bool skipWhitespace, bool skipSectionEnding, bool newLineIsSectionEnding)
{
    DbcToken *token;
    if (!(token = readToken(tokens, type, skipWhitespace, skipSectionEnding, newLineIsSectionEnding))) {
        return false;
    }

    if (data) {
        data->clear();
        data->append(token->getData());
    }

    free(token);
    return true;
}

bool 
Reader::recoverFromError( TokenType skipUntilToken )
{
   int errorCount = int(errors_.size());
   Token skip;
   for (;;)
   {
      if ( !readToken(skip) )
         errors_.resize( errorCount ); // discard errors caused by recovery
      if ( skip.type_ == skipUntilToken  ||  skip.type_ == tokenEndOfStream )
         break;
   }
   errors_.resize( errorCount );
   return false;
}

static void parseConstant (bool local)
{
	tokenInfo *const name = newToken ();
	readToken (name);
	if (local)
	{
		makeVhdlTag (name, VHDLTAG_LOCAL);
	}
	else
	{
		makeVhdlTag (name, VHDLTAG_CONSTANT);
	}
	skipToCharacterInInputFile (';');
	deleteToken (name);
}

static void parseLocal (tokenInfo *const token)
{
    Assert (isKeyword (token, KEYWORD_local));
    readToken (token);

    /*  Check keyword first in case local clause is empty
     */
    while (! isKeyword (token, KEYWORD_do)  &&
	   ! isKeyword (token, KEYWORD_once))
    {
#ifndef TYPE_REFERENCE_TOOL
	if (isType (token, TOKEN_IDENTIFIER))
	    makeEiffelLocalTag (token);

#endif
	readToken (token);
	if (isType (token, TOKEN_COLON))
	{
	    readToken (token);
	    if (isType (token, TOKEN_IDENTIFIER))
		parseType (token);
	}
    }
}

static void findJsonTags (void)
{
	tokenInfo *const token = newToken ();

	/* We allow multiple top-level elements, although it's not actually valid
	 * JSON.  An interesting side effect of this is that we allow a leading
	 * Unicode BOM mark -- even though ok, many JSON parsers will choke on it */
	do
	{
		readToken (token);
		parseValue (token);
	}
	while (token->type != TOKEN_EOF);

	deleteToken (token);
}

static void parseInherit (tokenInfo *const token)
{
    Assert (isKeyword (token, KEYWORD_inherit));
#ifdef TYPE_REFERENCE_TOOL
    readToken (token);
    while (isType (token, TOKEN_IDENTIFIER))
    {
        parseType (token);
        if (isType (token, TOKEN_KEYWORD))
        {
            switch (token->keyword)  /* check for feature adaptation */
            {
            case KEYWORD_rename:
            case KEYWORD_export:
            case KEYWORD_undefine:
            case KEYWORD_redefine:
            case KEYWORD_select:
                findKeyword (token, KEYWORD_end);
                readToken (token);
            default:
                break;
            }
        }
    }
#else
    readToken (token);
    while (isType (token, TOKEN_IDENTIFIER))
    {
        parseType (token);
        switch (token->keyword)  /* check for feature adaptation */
        {
        case KEYWORD_rename:
            parseRename (token);
            if (isKeyword (token, KEYWORD_end))
                readToken (token);
            break;

        case KEYWORD_export:
        case KEYWORD_undefine:
        case KEYWORD_redefine:
        case KEYWORD_select:
            findKeyword (token, KEYWORD_end);
            readToken (token);
            break;

        case KEYWORD_end:
            readToken (token);
            break;

        default:
            break;
        }
    }
#endif
}

static void findHtmlTags (void)
{
	tokenInfo token;

	token.string = vStringNew ();

	do
	{
		readToken (&token, true);
		if (token.type == TOKEN_TAG_START)
			readTag (&token, NULL, 0);
	}
	while (token.type != TOKEN_EOF);

	vStringDelete (token.string);
}

int Parser::readToken(s_cursor_t &it, const s_cursor_t &end) {
    // skipping any whitespace
    while (it != end && isspace(_lastChar))
      _lastChar = *(it++); // getting char and advancing cursor

    if (isalpha(_lastChar)) { /// identifier: [a-zA-Z][a-zA-Z0-9]*
        _identifier = _lastChar;
        while (it != end && isalnum((_lastChar = *(it++))))
          _identifier += _lastChar;

        if (_identifier == def_id)
            return tok_def;
        else if (_identifier == extern_id)
            return tok_extern;

        return tok_identifier;
    }

    if (isdigit(_lastChar)) { // number [0-9]
        std::string num;
        do {
          num += _lastChar;
        } while (it != end && isdigit((_lastChar = *(it++))));
        _number = strtol(num.c_str(), nullptr, 10);

        return tok_number;
    }

    if (_lastChar == '#') {
        // just a loop for ignoring everything after the #
        do
            _lastChar = *(it++);
        while (it != end && _lastChar != '\n' && _lastChar != '\r');

        if (_lastChar != EOF)
        return readToken(it, end);
    }

    if (it == end || _lastChar == EOF) {
      _lastChar = tok_eof;
    }

    // don't know what to do with this char
    int thisChar = _lastChar;
    _lastChar = *(it++);
    return thisChar;
}