C++ Tokenizer::eol方法代码示例

void Line::infToPost(Tokenizer& tokens, string& errs)
/*****************************************************************************/
{
  stack<Operator*> ops;

  //loop through all tokens and handle them
  for ( ; !tokens.eol(); tokens.nextToken()) {
    if (tokens.token().type() == FUNC) {
      process_function(tokens, errs);
      ++_tempSize; //for the return value
    }
    else if (tokens.token().type() == DECL) {
      process_newvar(tokens, errs);
    }
    else if (tokens.token().type() == CONSTANT) {
      process_number(tokens);
    }
    else if (tokens.token().type() == VAR) {
      process_existing_var(tokens, errs);
    }
    else if (tokens.token().type() == OPERATOR) {
      process_operator(tokens, ops);
    }
    else if (tokens.token().type() == SEMICOLON ||
	     tokens.token().type() == OPENBRACE) {
      tokens.nextToken();
      assert(tokens.eol());
      break;
    }
    else {
      CHECKERR(true, syntax_err(tokens.token().value()))
    }
    if (errs != "") return;
  }

  //put remaining opps at end of postfixLine
  while (!ops.empty()) {
    addNewObject(ops.top());
    ops.pop();
  }

  compile(errs, tokens);
  performNumericOps();
}

void Line::process_newvar(Tokenizer& tokens, string& errs)
/*****************************************************************************/
{
  string type = tokens.token().value();
  tokens.nextToken();

  CHECKERR(tokens.eol() || tokens.token().type() != VAR, var_err())

  string name = tokens.token().value();
  tokens.nextToken();

  bool isArray = false;
  Line* sizePtr = NULL;

  //if the following token is an open index, we know that our new variable
  //is an array
  if (!tokens.eol() && tokens.token().type() == OPENINDEX) {
    vector<Token> size_expr;
    tokens.nextToken(); //move past openindex

    //get all the tokens that are part of the array's size expression
    while (!tokens.eol() && tokens.token().type() != CLOSEINDEX) {
      size_expr.push_back(tokens.token());
      tokens.nextToken();
    }
    CHECKERR ((size_expr.size() == 0), ara_err(name))
    CHECKERR ((tokens.token().type() != CLOSEINDEX), ara_err(name))

    isArray = true;
    Tokenizer tempToken(size_expr);
    sizePtr = new Line(tempToken, _parent, errs, true);
  }
  else
    tokens.previousToken();

  if (_parent->getVar(name) == NULL)
    add_newvar(type, name, sizePtr, isArray);
  else
    { CHECKERR(true, dec_err(name)) }
}

void Line::process_existing_var(Tokenizer& tokens, string& errs)
/*****************************************************************************/
{
  string temp = tokens.token().value();
  Variable* v = _parent->getVar(temp);
  CHECKERR ((v == NULL), und_err(temp))

  //Note: we must allow for arrays to be passed to RTBoundFuncs without
  //having to use braces [].
  if (tokens.isArg()) {
    addNewObject(v);
    return;
  }

  //When we see an array variable, it must be followed by an index
  if (v->getObjectType() == ArrayVarOT) {
    tokens.nextToken();
    CHECKERR((tokens.eol() || tokens.token().type()!=OPENINDEX),ara_err(temp))
    tokens.nextToken(); //move past OPENINDEX
    vector<Token> index_list;

    //get all the tokens that are part of the array's index expression
    while (!tokens.eol() && tokens.token().type() != CLOSEINDEX) {
      index_list.push_back(tokens.token());
      tokens.nextToken();
    }
    CHECKERR ((index_list.size() == 0), ara_err(temp))
    CHECKERR ((tokens.eol()||tokens.token().type()!=CLOSEINDEX), ara_err(temp))

    Tokenizer tempToken(index_list);
    Line* indexPtr = new Line(tempToken, _parent, errs, true);
    ArrayIndex* ai = new ArrayIndex(v, indexPtr);
    addNewObject(ai);
  }
  else {
    addNewObject(v);
  }
}

void Line::process_function(Tokenizer& tokens, string& errs)
/*****************************************************************************/
{
  //create a function call object
  string temp = tokens.token().value();
  FunctionCall* funcCall = Registrar::generateCall(temp);
  CHECKERR((funcCall == NULL), func_err(temp))

  tokens.nextToken(); //move past funcname
  tokens.nextToken(); //move past open paren

  //put tokens into the argument lists.. We loop until we have seen the paren
  //that terminates this function call or until we have run out of tokens on
  //this line
  list< vector<Token> > args;
  vector<Token> currArg;
  int depth = 0;
  while(!((tokens.token().value() == ")" && depth == 0) || tokens.eol())) {

    //if we see a comma at paren depth zero, we have just reached the end of an
    //argument
    if (tokens.token().type() == COMMA && depth == 0) {
      assert(!currArg.empty());
      args.push_back(currArg);
      currArg.clear();
    }
    else {
      currArg.push_back(tokens.token());

      if (tokens.token() == Token(OPERATOR, "(", 0))
        ++depth;
      if (tokens.token() == Token(OPERATOR, ")", 0))
        --depth;
    }
    tokens.nextToken();
  }
  if (!currArg.empty())
    args.push_back(currArg);

  CHECKERR(
      tokens.eol() || tokens.token().value() != ")",
      arg_err(temp)
      )

  if (funcCall->hasVariableArgs()) {
    CHECKERR (
        args.size() < funcCall->getNumArgs(),
        arg_err(temp)
        )
  } else {
    CHECKERR(
        args.size() != funcCall->getNumArgs(),
        arg_err(temp)
        )
  }

  //Construct a Line for each argument
  list< vector<Token> >::iterator arg_itr = args.begin();
  for ( ; arg_itr != args.end(); ++arg_itr) {
    CHECKERR (((*arg_itr).size() == 0), arg_err(temp))
    Tokenizer tempToken(*arg_itr);
    Line* arg = new Line(tempToken, _parent, errs, true);
    funcCall->fillArg(arg);
  }

  addNewObject(funcCall);
}