C++ TokenScanner::ignoreWhitespace方法代码示例

/**
 * Receive information from file, processes and displays the result on the screen.
 */
void processText(TokenScanner & scanner){
    int counterSentences = 0;
    int counterWords = 0;
    int counterSyllable = 0;
    double grade = 0.0;

    scanner.ignoreWhitespace(); // ignores spaces
    scanner.addWordCharacters("'"); // does part of the token is the character "'"

    // processes each token
    while (scanner.hasMoreTokens()){
        string token = scanner.nextToken();
        counterSentences += checkSentences(token);
        if(checkWords(token)){
            counterWords ++;
            counterSyllable += colSyllable(token);
        }
    }
    // If the file does not contain any words or sentences, assume that their one
    if(counterWords == 0 || counterSentences == 0) counterSentences = counterWords = 1;

    grade = getGrade(counterSentences, counterWords, counterSyllable);

    cout << "Words : " << counterWords << endl;
    cout << "Syllable : " << counterSyllable << endl;
    cout << "Sentences : " << counterSentences << endl;
    cout << "Grade : " << grade << endl;
}

/* Function: scanCPlusPlusTokens()
 * Sets scanner features. */
void scanCPlusPlusTokens(TokenScanner & scanner) {
    scanner.ignoreWhitespace();
    /* Return numbers */
    scanner.scanNumbers();
    scanner.addOperator("%");
    scanner.addOperator("/");
    scanner.addOperator("*");
    scanner.addOperator("(");
    scanner.addOperator(")");
}

void processLine(string line, Program & program, EvalState & state) {
	TokenScanner scanner;
	scanner.ignoreWhitespace();
	scanner.scanNumbers();
	scanner.setInput(line);
    string str=scanner.nextToken();
    if (scanner.getTokenType(str)==NUMBER) {
		
        int lineNumber=stringToInteger(str);
        string token=scanner.nextToken();
        scanner.saveToken(token);
        if (token!="") {
            program.addSourceLine(lineNumber, line);
            program.setParsedStatement(lineNumber, parseStatement(scanner));
        }
        else {
            program.removeSourceLine(lineNumber);
        }
		
		
    }
	
    else if (str=="LIST") {
        for (int i=program.getFirstLineNumber(); i!=-1; i=program.getNextLineNumber(i)) {
            cout << program.getSourceLine(i)<<endl;
        }
    }
    else if (str=="CLEAR") {
        program.clear();
    }
    else if(str=="QUIT") exit(0);
    else if (str=="HELP") cout<<"This is a minimal BASIC interpreter."<<endl;
	
    else if (str=="RUN") {
        int currentLine=program.getFirstLineNumber();
        state.setCurrentLine(currentLine);
        while (currentLine!=-1) {                       
            program.getParsedStatement(currentLine)->execute(state);
			
            if(currentLine!=state.getCurrentLine()) {
                currentLine=state.getCurrentLine();
            }
            else {
                currentLine=program.getNextLineNumber(currentLine);
                state.setCurrentLine(currentLine);
            }
        }
    }
	
	
}

void toAlpha(std::string word, Map<std::string, std::string>& symbolTable) {
	TokenScanner scanner;
	scanner.ignoreWhitespace();

	for(char i = 'A'; i < 'Z'; i++)
		scanner.addWordCharacters(symbolTable.get(charToString(i)));

	scanner.setInput(word);

	while(scanner.hasMoreTokens()) {
		std::string symbol = scanner.nextToken();
		cout << symbolTable.get(symbol);
	}
	cout << endl;
}

void processLine(string line, Program & program, EvalState & state) {
   TokenScanner scanner;
   scanner.ignoreWhitespace();
   scanner.scanNumbers();
   scanner.setInput(line);
//   Expression *exp = parseExp(scanner);
//   int value = exp->eval(state);
//   cout << value << endl;
//   delete exp;
   if (!scanner.hasMoreTokens()) {
       return;
   }
   string firstToken = toUpperCase(scanner.nextToken());
   if (firstToken == "RUN") {
       runProgram(program, state);
   }
   else if (firstToken == "QUIT") {
       exit(0);
   }
   else if (firstToken == "HELP") {
       dspHelp();
   }
   else if (firstToken == "CLEAR") {
       program.clear();
   }
   else if (firstToken == "LIST") {
       listCmd(program);
   }
   else if (firstToken == "INPUT" || firstToken == "PRINT" || firstToken == "LET") {
       scanner.saveToken(firstToken);
       Statement * stmt = parseStatement(scanner);
       stmt->execute(state);
       delete stmt;
   }
   else {
       int lineNumber = stringToInteger(firstToken);
       if (!scanner.hasMoreTokens()) {
           program.removeSourceLine(lineNumber);
       }
       else {
           int length = firstToken.length();
           string source = line.substr(length);
           program.addSourceLine(lineNumber, source);
           Statement * stmt = parseStatement(scanner);
           program.setParsedStatement(lineNumber, stmt);
       }
   }
}

void interpreter(void) {
	std::string line;
	TokenScanner scanner;
	scanner.ignoreWhitespace();
	scanner.scanNumbers();
	Map<std::string, std::string> symbolTable;

	cout << "Interpreter is running: " << endl;

	// Main loop:
	std::string key, value;
	while(true) {
		cout << "> ";
		line = getLine();

		if(line == "quit")
			break;

		else if(line == "list") {
			for(std::string key : symbolTable) {
				cout << key << " = " << symbolTable.get(key) << endl;
			}
		}

		else if(symbolTable.containsKey(line))
			cout << symbolTable.get(line) << endl;

		// Parse statements:
		else {
			scanner.setInput(line);
			bool isValue = false;
			while(scanner.hasMoreTokens()) {
				std::string token = scanner.nextToken();
				if(isValue)
					value = token;
				else if(token == "=")
					isValue = true;
				else
					key = token;
			}
			symbolTable.add(key, value);
		}
	}

	cout << "Interpreter is stopped." << endl;
}

void processLine(string line, Program & program, EvaluationContext & context) {
   TokenScanner scanner;
   scanner.ignoreWhitespace();
   scanner.setInput(line);
    
    string firstToken = toUpperCase(scanner.nextToken());
    if (firstToken == "RUN") {
        runProgram(program, context);
    }
    else if (firstToken == "LIST") {
        listProgram(program, scanner);
    }
    else if (firstToken == "CLEAR") {
        program.clear();
    }
    else if (firstToken == "HELP") {
        if (!scanner.hasMoreTokens()) {
            displayHelp();
        }
        else {
            specificHelp(scanner);
        }
    }
    else if (firstToken == "QUIT") {
        exit(0);
    }
    else {
        int lineNumber = stringToInteger(firstToken);
        if (!scanner.hasMoreTokens()) {
            program.removeSourceLine(lineNumber);
        }
        else {
            Statement *stmt = parseStatement(scanner);
            program.addSourceLine(lineNumber, line);
            program.setParsedStatement(lineNumber, stmt);
        }
    }
}

/*
 * Function: processLine
 * Usage: processLine(line, program, state);
 * -----------------------------------------
 * Processes a single line entered by the user.  In this version,
 * the implementation does exactly what the interpreter program
 * does in Chapter 19: read a line, parse it as an expression,
 * and then print the result.  In your implementation, you will
 * need to replace this method with one that can respond correctly
 * when the user enters a program line (which begins with a number)
 * or one of the BASIC commands, such as LIST or RUN.
 */
void processLine(string line, Program & program, EvalState & state) {
   TokenScanner scanner;
   scanner.ignoreWhitespace();
   scanner.scanNumbers();
   scanner.setInput(line);
   string token=scanner.nextToken();
   TokenType type=scanner.getTokenType(token);
   int lineNumb;
   int priorLineNumb;
   if(type==NUMBER){
     if(scanner.hasMoreTokens()){
       program.addSourceLine(stringToInteger(token),line);
       Statement *stmt=parseStatement(scanner); 
       program.setParsedStatement(stringToInteger(token),stmt);  //link statement to line number
     }
     else
       program.removeSourceLine(stringToInteger(token));
   }
   if(type==WORD){
     if(token=="LIST"){
       lineNumb=program.getFirstLineNumber();
       while(lineNumb!=-1){
	 cout<<program.getSourceLine(lineNumb)<<endl;
	 lineNumb=program.getNextLineNumber(lineNumb);
       }
     }
     else if(token=="RUN"){
       state.setCurrentLine(program.getFirstLineNumber());
       lineNumb=state.getCurrentLine();
       while(lineNumb!=-1){	 
	 if(program.hasLineNumber(lineNumb)==false)
	   error("Invalid line reference at program line "+integerToString(priorLineNumb));
	 Statement *stmt=program.getParsedStatement(lineNumb);
	 state.setCurrentLine(program.getNextLineNumber(lineNumb));
	 stmt->execute(state);
	 priorLineNumb=lineNumb;
	 lineNumb=state.getCurrentLine();
       }
     }
     else if(token=="CLEAR"){
       program.clear();
     }
     else if(token=="HELP"){
       cout<<"Minimal Basic Statements"<<endl; 
       cout<<"REM   This statement used for comments"<<endl;
       cout<<"LET   This statement is BASIC's assigment statement"<<endl;
       cout<<"PRINT In minimal BASIC, the PRINT statement has the form: "<<endl;
       cout<<"         PRINT exp"<<endl;
       cout<<"      where exp is an expression."<<endl;
       cout<<"INPUT In the minimal version of the BASIC interpreter, the INPUT statement has the form:"<<endl;
       cout<<"         INPUT var"<<endl;
       cout<<"      where var is a variable read in from the user"<<endl;
       cout<<"GOTO  This statement has the syntax"<<endl;
       cout<<"         GOTO n"<<endl;
       cout<<"      which forces the program to continue from line n instead of continuing with the next statement"<<endl;
       cout<<"IF    This statement provides conditional control. The syntax for this statement is: "<<endl;
       cout<<"      IF exp1 op exp2 THEN n"<<endl;
       cout<<"      where exp1 and exp2 are expressions and op is one of the conditional operators =, <, or >."<<endl;
       cout<<"END   This statment marks the end of the program."<<endl;
       cout<<"Commands to control the BASIC interpreter"<<endl;
       cout<<"RUN   This command starts program execution beginning at the lowest-numbered line. "<<endl;
       cout<<"LIST  This command lists the steps in the program in numerical sequence."<<endl;
       cout<<"CLEAR This command deletes the program so the user can start entering a new one."<<endl;
       cout<<"HELP  This command provides a simple help message describing the interpreter."<<endl;
       cout<<"QUIT  Typing QUIT exits from the BASIC interpreter by calling exit(0)."<<endl;	 
     }
     else if(token=="QUIT"){
       exit(0);
     }
     else
       cout<<"Invalid command"<<endl;     
   }
}

/* main() manages the user interface;
 * instantiates priority queue object, then operates loop to read input 
 * from user and call the appropriate priority queue method
 */
int main() {
   PriorityQueue pq;
   TokenScanner scanner;
   while (true) {
      string line = getLine("> ");
      scanner.setInput(line);
      scanner.ignoreWhitespace();
      string cmd=scanner.nextToken();
      if (cmd == "help") {
         helpCommand();
      }      
      else if (cmd == "enqueue") {
	if(scanner.hasMoreTokens()){
	  string value=scanner.nextToken();
	  if(scanner.hasMoreTokens()){
	    scanner.scanNumbers();
	    string priorityStr=scanner.nextToken();
	    double priority=stringToDouble(priorityStr);	
	    pq.enqueue(value,priority);
	  }
	  else
	    pq.enqueue(value);
	}
      }    
      else if (cmd == "dequeue") {
	if(pq.isEmpty())
	  cout<<"The queue is empty"<<endl;
	else
	  cout<<pq.dequeue()<<endl;
      }
      else if (cmd == "peek") {
	if(pq.isEmpty())
	  cout<<"The queue is empty"<<endl;
	else
	  cout<<pq.peek()<<endl;
      }
      else if (cmd == "peekPriority"||cmd=="peekpriority") {
	if(pq.isEmpty())
	  cout<<"The queue is empty"<<endl;
	else
	  cout<<pq.peekPriority()<<endl;
      }
      else if (cmd == "clear") {
	pq.clear();
      }
      else if (cmd == "size") {
	cout<<pq.size()<<endl;
      }
      else if (cmd == "isEmpty"||cmd=="isempty") {
	if(pq.isEmpty())
	  cout<<"true";
	else
	  cout<<"false";
	cout<<endl;
      }
      else if(cmd=="list")
	list(pq);
      else {
         cout << "Undefined command: " << cmd << endl;
      }
   }
   return 0;
}

void processLine(string line, Program & program, EvalState & state) {
  /* TokenScanner scanner;
   scanner.ignoreWhitespace();
   scanner.scanNumbers();
   scanner.setInput(line);
   Expression *exp = parseExp(scanner);
   int value = exp->eval(state);
   cout << value << endl;
   delete exp;*/

   TokenScanner scanner;
   scanner.ignoreWhitespace();
   scanner.scanNumbers();
   scanner.setInput(line);
   string token = scanner.nextToken();
   TokenType type = scanner.getTokenType(token);
   if (type == WORD){
	   if (token == "RUN")
	   {
		   int current_line=program.getFirstLineNumber();
		   while (current_line>=0)
		   {
			   int next_line = program.getNextLineNumber(current_line);
			   Statement* stat=program.getParsedStatement(current_line);
			   StatementType type = stat->get_type();
			   if (type == END)
				   break;
			   if (type == GOTO)
			   {
				   ((Gotostmt*)stat)->execute(state,current_line);
				   
				   continue;
			   }
			   if (type == IF)
			   {
				   ((Ifstmt*)stat)->execute(state, current_line);
				   if (current_line != -2)
			  	       continue;
			   }
			  
			   stat->execute(state);
			   current_line = next_line;
		   }
		  
	   }
	   if (token == "QUIT")
	   {
		   exit(0);
	   }
	   if (token == "CLEAR")
	   {
		   program.clear();
		   EvalState state1;
		   state = state1;
		   
	   }
	   if (token == "HELP")
	   {
	   }
	   if (token == "LIST")
	   {
		   int current_line = program.getFirstLineNumber();
		   while (current_line != -1)
		   {
			   cout << program.getSourceLine(current_line) << endl;
			   current_line = program.getNextLineNumber(current_line);
		   }
	   }
	   if (token == "INPUT"){
		   string st = scanner.nextToken();
		   if (scanner.hasMoreTokens())
			   error("SYNTAX ERROR");
		   Inputstmt inputst(st);
		   inputst.execute(state);
		  
	   }
	   if (token == "PRINT"){
		   Printstmt printst(scanner);
		   printst.execute(state);
	   }

	   if (token == "LET")
	   {
		   Letstmt lets(scanner);
		   lets.execute(state);
		 
	   }
	   if (token != "RUN"&&token != "CLEAR"&&token != "HELP"&&token != "LIST"&&token != "QUIT"&&token != "PRINT"&&token != "LET"&&token != "INPUT")
		   error("SYNTAX ERROR");
   }


   else if (type == NUMBER){
	   int line_num = atoi(token.c_str());
	   
	   string token2 = scanner.nextToken();
	   if (token2 == "")
	   {
		   program.removeSourceLine(line_num);
		   return;
//.........这里部分代码省略.........