C++ TokenList::AddToken方法代码示例

void ParseExpression(TokenList &a_list, const std::string &a_expression)
{
	enum class ParseState {None, Number, Name} state = ParseState::None;
	int elen = a_expression.length() + 1;
	int idx = 0;
	int tstart = 0;
	char c_char, n_char;

	while (true)
	{
		if (idx < elen)
		{
			c_char = a_expression[idx];
			if (idx + 1 < elen) n_char = a_expression[idx + 1];
			else n_char = 0;
			idx++;
		}
		else
		{
			if (state == ParseState::Name)
			{
				a_list.AddToken(ExtractValue(a_expression, tstart, idx - 1, Token::Type::Name));
				a_list.tail->index = tstart + 1;
			}
			else if (state == ParseState::Number)
			{
				a_list.AddToken(ExtractValue(a_expression, tstart, idx - 1, Token::Type::Number));
				a_list.tail->index = tstart + 1;
			}
			break;
		}

		if (std::isspace(c_char) && state != ParseState::None)
		{
			a_list.AddToken(ExtractValue(a_expression, tstart, idx - 1, (state == ParseState::Name ? Token::Type::Name : Token::Type::Number)));
			a_list.tail->index = tstart + 1;
			state = ParseState::None;
		}
		else if (isop(c_char))
		{
			if (state != ParseState::None)
			{
				a_list.AddToken(ExtractValue(a_expression, tstart, idx - 1, (state == ParseState::Name ? Token::Type::Name : Token::Type::Number)));
				a_list.tail->index = tstart + 1;
			}

			switch (c_char)
			{
			case '+': a_list.AddToken(new Token(Token::OpType::Add, idx)); break;
			case '-':
				if (a_list.count == 0 || a_list.tail->optype != Token::OpType::None)
					a_list.AddToken(new Token(Token::OpType::Unary, idx));
				else a_list.AddToken(new Token(Token::OpType::Sub, idx));
				break;
			case '*': a_list.AddToken(new Token(Token::OpType::Mul, idx)); break;
			case '/': a_list.AddToken(new Token(Token::OpType::Div, idx)); break;
			case '^': a_list.AddToken(new Token(Token::OpType::Pow, idx)); break;
			case '%': a_list.AddToken(new Token(Token::OpType::Mod, idx)); break;
			case '(': a_list.AddToken(new Token(Token::OpType::LPar, idx)); break;
			case ')': a_list.AddToken(new Token(Token::OpType::RPar, idx)); break;
			case ',': a_list.AddToken(new Token(Token::OpType::Comma, idx)); break;
			case '<':
				if (n_char == '=')
				{
					a_list.AddToken(new Token(Token::OpType::LE, idx));
					idx++;
				}
				else a_list.AddToken(new Token(Token::OpType::LT, idx));
				break;
			case '>':
				if (n_char == '=')
				{
					a_list.AddToken(new Token(Token::OpType::GE, idx));
					idx++;
				}
				else a_list.AddToken(new Token(Token::OpType::GT, idx));
				break;
			case '=':
				if (n_char == '=')
				{
					a_list.AddToken(new Token(Token::OpType::EQ, idx));
					idx++;
				}
				break;
			case '!':
				if (n_char == '=')
				{
					a_list.AddToken(new Token(Token::OpType::NE, idx));
					idx++;
				}
				break;
			}
			state = ParseState::None;
		}
		else if (issp(c_char) && state == ParseState::None && !isalpha(n_char))
		{
			switch (c_char)
			{
			case 't': a_list.AddToken(new Token(Token::SpecialType::Time, idx)); break;
			case 'x': a_list.AddToken(new Token(Token::SpecialType::XAxis, idx)); break;
//.........这里部分代码省略.........

TokenList BracketMatcher::ReadCombination()
{
    // TODO: split into smaller functions

    enum EMode
    {
          eOther
        , eOperator
        , eFirstOperand
    };

    TokenList ret;
    unsigned int bracket_level = 0;
    EMode mode = eOther;

    while( true )
    {
        Token token = lexer_.NextToken();

        // If the stream ended before a combo was closed, return an empty
        // list - the user pressed Ctrl-D and doesn't want any output.
        if( token.IsEndOfStream() )
        {
            return TokenList();
        }

        token.AddToColumn( newline_processor_.GetIndent() );

        // If we have got to the first operand of a combination
        // store the indent level so that we can indent correctly
        // in the repl when the user presses return.
        switch( mode )
        {
            case eOperator:
            {
                mode = eFirstOperand;
                break;
            }
            case eFirstOperand:
            {
                newline_processor_.PushIndent( bracket_level, token.Column() );
                mode = eOther;
                break;
            }
            default:
            {
                break;
            }
        }

        ret.AddToken( token );

        // If we've finished this combination, return now
        if( token.Name() == ")" && bracket_level == 1 )
        {
            break;
        }

        // Otherwise, adjust our bracket level if there is a ( or )
        if( token.Name() == "(" )
        {
            ++bracket_level;
            mode = eOperator; // The next token will be the operator
        }
        else if( token.Name() == ")" )
        {
            newline_processor_.PopIndent( bracket_level );
            --bracket_level;
            mode = eOther;
        }

        // If we got a bare symbol (no brackets) we exit here
        // (except that we skip straight past quotes)
        if( bracket_level == 0 && token.Name() != "'" )
        {
            break;
        }

        // If the lexer's token ended with newline, it couldn't call
        // newline (because we hadn't processed the token yet).  We
        // have now processed it, so we can call NewLine now.
        if( print_continuation_ && lexer_.EndedWithNewLine() )
        {
            newline_processor_.NewLine();
        }
    }

    // When the combination (or bare symbol) is finished, we don't
    // need to track indentation any more, so reset the newline processor,
    // and we don't care about any newlines before the next token, so
    // we skip whitespace in the lexer.
    newline_processor_.Reset();

    return ret;
}