本文整理汇总了C++中TokenStream::addToken方法的典型用法代码示例。如果您正苦于以下问题：C++ TokenStream::addToken方法的具体用法？C++ TokenStream::addToken怎么用？C++ TokenStream::addToken使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类TokenStream的用法示例。
在下文中一共展示了TokenStream::addToken方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C++代码示例。
示例1: lex
TokenStream Scanner::lex(std::istream &programStream) {
TokenStream tokens;
unsigned line = 1;
unsigned col = 1;
ScanState state = ScanState::INITIAL;
while (programStream.peek() != EOF) {
if (programStream.peek() == '_' || std::isalpha(programStream.peek())) {
std::string tokenString(1, programStream.get());
while (programStream.peek() == '_' ||
std::isalnum(programStream.peek())) {
tokenString += programStream.get();
}
Token newToken;
newToken.type = getTokenType(tokenString);
newToken.lineBegin = line;
newToken.colBegin = col;
newToken.lineEnd = line;
newToken.colEnd = col + tokenString.length() - 1;
if (newToken.type == Token::Type::IDENT) {
newToken.str = tokenString;
}
tokens.addToken(newToken);
col += tokenString.length();
} else {
switch (programStream.peek()) {
case '(':
programStream.get();
tokens.addToken(Token::Type::LP, line, col++);
break;
case ')':
programStream.get();
tokens.addToken(Token::Type::RP, line, col++);
break;
case '[':
programStream.get();
tokens.addToken(Token::Type::LB, line, col++);
break;
case ']':
programStream.get();
tokens.addToken(Token::Type::RB, line, col++);
break;
case '{':
programStream.get();
tokens.addToken(Token::Type::LC, line, col++);
break;
case '}':
programStream.get();
tokens.addToken(Token::Type::RC, line, col++);
break;
case '<':
programStream.get();
if (programStream.peek() == '=') {
programStream.get();
tokens.addToken(Token::Type::LE, line, col, 2);
col += 2;
} else {
tokens.addToken(Token::Type::LA, line, col++);
}
break;
case '>':
programStream.get();
if (programStream.peek() == '=') {
programStream.get();
tokens.addToken(Token::Type::GE, line, col, 2);
col += 2;
} else {
tokens.addToken(Token::Type::RA, line, col++);
}
break;
case ',':
programStream.get();
tokens.addToken(Token::Type::COMMA, line, col++);
break;
case '.':
programStream.get();
switch (programStream.peek()) {
case '*':
programStream.get();
tokens.addToken(Token::Type::DOTSTAR, line, col, 2);
col += 2;
break;
case '/':
programStream.get();
tokens.addToken(Token::Type::DOTSLASH, line, col, 2);
col += 2;
break;
default:
tokens.addToken(Token::Type::PERIOD, line, col++);
break;
}
break;
case ':':
programStream.get();
tokens.addToken(Token::Type::COL, line, col++);
break;
case ';':
//.........这里部分代码省略.........