C++ TokenStream类代码示例

TypeRef Parse_Type_ErasedType(TokenStream& lex, bool allow_trait_list)
{
    Token   tok;

    auto ps = lex.start_span();
    ::std::vector<Type_TraitPath>   traits;
    ::std::vector<AST::LifetimeRef>   lifetimes;
    do {
        if( LOOK_AHEAD(lex) == TOK_LIFETIME ) {
            GET_TOK(tok, lex);
            lifetimes.push_back(AST::LifetimeRef( /*lex.point_span(),*/ lex.get_ident(mv$(tok)) ));
        }
        else
        {
            AST::HigherRankedBounds hrbs;
            if( lex.lookahead(0) == TOK_RWORD_FOR )
            {
                hrbs = Parse_HRB(lex);
            }
            traits.push_back({ mv$(hrbs), Parse_Path(lex, PATH_GENERIC_TYPE) });
        }
    } while( GET_TOK(tok, lex) == TOK_PLUS );
    PUTBACK(tok, lex);

    return TypeRef(lex.end_span(mv$(ps)), TypeData::make_ErasedType({ mv$(traits), mv$(lifetimes) }));
}

IntegerType
parse_ascii_integer(TokenStream & stream) {
  if (!stream.peek()) throw std::runtime_error("eof");

  if (!ascii_digit_value(*stream.peek())) {
    throw std::runtime_error("not at a number!");
  }

  IntegerType toret = 0;
  while (true) {
    auto inb = stream.peek();
    if (!inb) break;
    auto digit_val = ascii_digit_value(*inb);
    if (!digit_val) break;

    auto oldtoret = toret;
    toret = toret * 10 + *digit_val;
    if (toret < oldtoret) {
      throw std::runtime_error("overflow occured");
    }

    stream.skip();
  }

  return toret;
}

// Macro-expand a macro argument 'arg' to create 'expandedArg'.
// Does not replace 'arg'.
// Returns nullptr if no expanded argument is created.
TPpContext::TokenStream* TPpContext::PrescanMacroArg(TokenStream& arg, TPpToken* ppToken, bool newLineOkay)
{
    // expand the argument
    TokenStream* expandedArg = new TokenStream;
    pushInput(new tMarkerInput(this));
    pushTokenStreamInput(arg);
    int token;
    while ((token = scanToken(ppToken)) != tMarkerInput::marker && token != EndOfInput) {
        token = tokenPaste(token, *ppToken);
        if (token == tMarkerInput::marker || token == EndOfInput)
            break;
        if (token == PpAtomIdentifier && MacroExpand(ppToken, false, newLineOkay) != 0)
            continue;
        expandedArg->putToken(token, ppToken);
    }

    if (token == EndOfInput) {
        // MacroExpand ate the marker, so had bad input, recover
        delete expandedArg;
        expandedArg = nullptr;
    } else {
        // remove the marker
        popInput();
    }

    return expandedArg;
}

void calculate()
{
    const char quit = 'q';       // t.kind==quit means that t is a quit Token
    const char print = ';';      // t.kind==print means that t is a print Token
    const std::string prompt = "> ";
    const std::string equals = "= ";  // used to indicate that what follows is an evaluation
    TokenStream tokenStream;
    std::string enterToClose = "~~";

    while (std::cin)
        try
        {
            std::cout << prompt;
            Token token = tokenStream.getToken();

            if (token.kind == print)
                token = tokenStream.getToken();   // eat ';'

            if (token.kind == quit)
                break;

            tokenStream.setToken(token);      // reset token into tokenStream
            std::cout << equals << expression(tokenStream) << '\n';
        }

        catch (std::exception& e)
        {
            std::cerr << e.what() << '\n';                      // write error message
            tokenStream.ignore(print);
        }
}

        Option<Term> singleTerm(TokenStream& tokens) {
            if (tokens->type == Token::LAMBDA) {
                return lambdaTerm(tokens);

            } else if (tokens->type == Token::OPEN_BRACKET) {
                tokens.advance();
                Option<Term> t = term(tokens);
                if (tokens->type == Token::CLOSE_BRACKET) {
                    tokens.advance();
                    return t;
                } else return errorTerm("expected closing bracket");

            } else if (tokens->type == Token::IDENTIFIER) {
                string id = tokens->identifierValue;
                tokens.advance();
                return Option<Term>(variableTerm(id));

            } else if (tokens->type == Token::INTEGER) {
                int value = tokens->intValue;
                tokens.advance();
                return Option<Term>(integerTerm(value));

            } else if (tokens->type == Token::CLOSE_BRACKET) {
                return errorTerm("unexpected closing bracket");

            } else if (!tokens.good()) {
                return errorTerm("expected a term");
            } else return errorTerm("unknown token");
        }

double expression(TokenStream& tokenStream)
{
    double result = term(tokenStream);       // set result to value of term
    Token token = tokenStream.getToken();  // get next token from tokenStream

    while (true)
    {
        if (token.kind == '+')
        {
            result += term(tokenStream);      // evaluate term and add to result
            token = tokenStream.getToken(); // get next token from tokenStream
        }

        if (token.kind == '-')
        {
            result -= term(tokenStream);      // evaluate term and subtract from result
            token = tokenStream.getToken(); // get next token from tokenStream
        }

        if (token.kind != '+' && token.kind != '-')
        {
            tokenStream.setToken(token);    // reset token into token stream
            return result;                    // return result
        }
    }
}

SAWYER_EXPORT std::string
Grammar::evalFunction(TokenStream &tokens, ErrorLocation &eloc) const {
    ASSERT_require(tokens.isa(TOK_FUNCTION));
    std::string funcName = tokens.lexeme();
    ASSERT_require(funcName.size() >= 2 && '@' == funcName[0]);
    funcName = funcName.substr(1);
    tokens.consume();

    // Get the function declaration
    const Function::Ptr func = functions_.getOrDefault(funcName);
    if (!func)
        throw SyntaxError("function \"" + funcName + "\" is not declared");

    // Parse the actual arguments
    std::vector<std::string> actuals;
    while (tokens.isa(TOK_LEFT)) {
        tokens.consume();
        if (func->isMacro()) {
            actuals.push_back(readArgument(tokens, eloc, CONSUME));
        } else {
            actuals.push_back(evalArgument(tokens, eloc, CONSUME));
        }
    }
    func->validateArgs(actuals, tokens);

    ErrorLocation::Trap t(eloc, tokens, "in function \"" + funcName + "\"");
    std::string retval = func->eval(*this, actuals);
    t.passed();
    return retval;
}

int main(int argc, char *argv[]){

  if (argc != 2){

    cout << "Usage: " << argv[0] << " <filename>" << endl;
    exit (1);
  }
  else {

    try {
      FileReader *reader =  _CLNEW FileReader(argv[1],"UTF-8");
      
      Analyzer *analyzer = new StandardAnalyzer();
      TokenStream *tokenStream = analyzer->tokenStream(_T("iets"), reader);
      Token token;
      while (tokenStream->next(&token)){
	
	char buffer[1000];
	STRCPY_TtoA(buffer, token.termText(), 1000);
	cout << "\t token: " << buffer;
	STRCPY_TtoA(buffer, token.type(), 1000);
	
	cout << " type: " << buffer << endl;
      }
      tokenStream->close();
      delete tokenStream;
      //reader->close();
      delete reader;
      delete analyzer;
    } catch (CLuceneError &e){

      cerr << e.what() << endl;
    }
  }
}

void print_remaining_tokens(std::ostream& out, TokenStream& tokens)
{
    for (int i=0; i < tokens.remaining(); i++) {
        if (i != 0) out << " ";
        out << get_token_text(tokens.next(i).match);
        out << "(" << tokens.nextStr(i) << ")";
    }
}

void
skip_token(TokenStream & stream, Token token) {
  while (true) {
    auto inb = stream.peek();
    if (!inb || *inb != token) break;
    stream.skip();
  }
}

void
expect(TokenStream & stream, Token expected) {
  auto maybe_token = stream.peek();
  if (!maybe_token || *maybe_token != expected) {
    throw std::runtime_error("Failed expect");
  }
  stream.skip();
}

static bool
SetDisplayURL(ExclusiveContext *cx, TokenStream &tokenStream, ScriptSource *ss)
{
    if (tokenStream.hasDisplayURL()) {
        if (!ss->setDisplayURL(cx, tokenStream.displayURL()))
            return false;
    }
    return true;
}

static bool
SetSourceMap(JSContext *cx, TokenStream &tokenStream, ScriptSource *ss, JSScript *script)
{
    if (tokenStream.hasSourceMap()) {
        if (!ss->setSourceMap(cx, tokenStream.releaseSourceMap(), script->filename()))
            return false;
    }
    return true;
}

static bool
SetSourceMap(ExclusiveContext *cx, TokenStream &tokenStream, ScriptSource *ss)
{
    if (tokenStream.hasSourceMap()) {
        if (!ss->setSourceMap(cx, tokenStream.releaseSourceMap()))
            return false;
    }
    return true;
}

static bool
SetSourceMap(ExclusiveContext* cx, TokenStream& tokenStream, ScriptSource* ss)
{
    if (tokenStream.hasSourceMapURL()) {
        MOZ_ASSERT(!ss->hasSourceMapURL());
        if (!ss->setSourceMapURL(cx, tokenStream.sourceMapURL()))
            return false;
    }
    return true;
}