C++ Token::GetType方法代码示例

SymVarConst* Parser::ParseConst()
{
	SymType* type = NULL;
	string Value = "";
	while (true)
	{
		if(t.GetValue() == "+" || t.GetValue() == "-")
		{
			Value = t.GetValue() + Value;
			t = sc.GetNextToken();
		}
		else
			if(t.GetType() == int_const_dec)
				return new SymVarConst(Value + t.GetValue(), Value + t.GetValue(), (SymType*)mainTable->find("integer")->second);
			else
				if (t.GetType() == float_const)
					return new SymVarConst(Value + t.GetValue(), Value + t.GetValue(), (SymType*)mainTable->find("real")->second);
				else
				{
					bool found = false;
					_Table::iterator it = TableStack.Find(t.GetValue(), found);
					if (t.GetType() != identifier)
						throw Error("wrongConst", t);
					if (!found)
						throw Error("NotConst", t);
					if (!it->second->IsConst())
						throw Error("wrongConstType",t);
					return new SymVarConst(t.GetValue(), Value + ((SymVarConst*)it->second)->GetValue(), ((SymVarConst*)it->second)->GetType());
				}
	}
}

int main()
{
	ifstream in;
	in.open("C:/Users/taoyanqi/Desktop/Êý¾Ý½á¹¹/ImproveLexer/ImproveLexer/aa.txt");
	Scanner scan(in);
	for (Token t; t.GetType() != END_OF_FILE;)
	{
		// cout << ( t = scan.NextToken() );
		t = scan.GetNextToken();
		cout << t.GetRow() << ':' << t.GetColumn() << '\t' << TOKEN_DESCRIPTION[t.GetType()]
			<< '\t' << TOKEN_VALUE_DESCRIPTION[t.GetValue()] << '\t' << t.GetName() << endl;
	}
}

NodeWrite* Parser::ParseWrite(bool isFirst, bool isWriteln)
{
	NodeWrite* ans = NULL;
	t = sc.GetNextToken();
	if (isFirst)
		t = RequireToken("(", "\"(\" expected");
	if (t.GetType() == string_const)
	{
		ans = new NodeWriteStr(STR, "\""+t.GetValue().substr(1, t.GetValue().size()-2)+"\"");
		t = sc.GetNextToken();
	}
	else
	{
		NodeExpression* exp = ParseComparision();
		OutputType T;
		if (exp->GetType()->IsInt())
			T = INT;
		else
			T = FLOAT;
		ans = new NodeWriteExp(T, exp);
	}
	if (t.GetValue() == ",")
		ans->SetNext(ParseWrite(false, isWriteln));
	else
	{
		t = RequireToken(")", "\")\" expected");
		if (isWriteln)
			ans->SetNext(new NodeWriteln());
	}
	return ans;
}

int FAO::GetPrecedence()
{
	if (cur_token_.GetType() == TokenType::TOKEN_OPERATION){
		return this->binary_op_precedence_[cur_token_.GetOperation()];
	}
	return -1;
}

// ****************************************************************************
//  Method:  ExprParser::Parse
//
//  Purpose:
//    Main external routine for clients.
//    Re-initialize the parser and scanner, and process one at a time until
//    we get an error or acceptance.  Return the parse tree when finished.
//
//  Programmer:  Jeremy Meredith
//  Creation:    April  5, 2002
//
//  Modifications:
//    Jeremy Meredith, Mon Jul 28 16:13:35 PDT 2003
//    Made it report normal parse errors through the viewer window
//    mechanism if needed.  Unhandled reductions will only occur during
//    debugging and can still go to cerr.
//
//    Hank Childs, Fri Aug  8 08:13:21 PDT 2003
//    Have error messages be issued in a way that it is independent of 
//    component.
//
//    Jeremy Meredith, Fri Aug 15 12:49:01 PDT 2003
//    Added the EMT_EXCEPTION type, and renamed EMT_VIEWER to EMT_COMPONENT.
//
//    Jeremy Meredith, Wed Nov 24 11:51:59 PST 2004
//    Refactored.  There's a new base class for the ExprParser and the
//    return types became more general.
//
//    Jeremy Meredith, Tue Dec 28 11:18:37 PST 2004
//    Made the current text a data member to help with macro support.
//
//    Hank Childs, Tue Dec 28 16:14:47 PST 2004
//    Rename GetText and PrintText to GetErrorText and PrintErrorText.
//
//    Hank Childs, Fri Jan 28 14:07:18 PST 2005
//    Use exception macros.
//
//    Jeremy Meredith, Mon Jun 13 16:17:14 PDT 2005
//    Delete the tokens that have not taken part in a rule reduction -- in 
//    this case that means Space tokens and the final EOF token.
//
// ****************************************************************************
ParseTreeNode*
ExprParser::Parse(const std::string &s)
{
    text = s;

    // Change weird spacing (tabs, newlines) to normal ones.
    // This will make printing error messages later much easier.
    for (size_t i=0; i<text.length(); i++)
        if (text[i] == '\t' || text[i] == '\n') text[i] = ' ';

    TRY
    {
        Init();
        scanner.SetInput(text);
        Token *token = NULL;
        while (!Accept()) 
        {
            token = scanner.ScanOneToken();
            if (token->GetType() != TT_Space)
                ParseOneToken(token);
            else
                delete token;
        }
        delete token;
    }
    CATCH2(UnhandledReductionException, e)
    {
        // This should only occur during debugging; print to cerr anyway
        cerr << e.Message() << endl;
        cerr << "Rule = " << *(e.GetRule()) << endl;
        e.GetPos().PrintErrorText(cerr, text);
        CATCH_RETURN2(1, NULL);
    }

int CompilerMain::GetNextToken()
{
	
	Token* token = tokenList->at(nextTokenIndex);
	yylval = token->GetValue().toStdString();
	//qDebug()<<("Next Token is " + token->GetValue());
    nextTokenIndex++;
    return token->GetType();
}

ExprAST * FAO::ParsePrimary()
{
	switch (cur_token_.GetType()){
	case TokenType::TOKEN_INTEGER:
		return ParseNumberExpr(); 
	case TokenType::TOKEN_LEFT_PARAM:
		return ParseParentExpr();
	default:
		return Error("unknow token when expecting an expression");
	}
}

ExprAST * FAO::ParseParentExpr()
{
	Advance(); // eat "("
	ExprAST * ret = ParseExpression();
	if (ret == nullptr)return 0;

	if (cur_token_.GetType() != TokenType::TOKEN_RIGHT_PARAM){
		return Error("expected ')'");
	}
	Advance(); // eat ")"
	return ret;
}

SymTable* Parser::ParseArguments(vector<string>* &arguments)
{
	SymTable* argTable = new SymTable;
	if (t.GetValue() != "(")
		return argTable;
	vector<string> argNames;
	bool isVar = false;
	do
	{
		t = sc.GetNextToken();
		if (t.GetValue() == "var")
		{
			isVar = true;
			t = sc.GetNextToken();
		}
		if (t.GetType() != identifier)
			throw Error("incorrect name of argument", t);
		argNames.push_back(t.GetValue());
		arguments->push_back(t.GetValue());
		t = sc.GetNextToken();
		if (t.GetValue() == ",")
			continue;
		if (t.GetValue() == ":")
		{
			SymType* Type = ParseType(false);
			if (IsExplType(Type->GetName()))
				throw Error("arguments type mismatch", t);
			t = sc.GetNextToken();
			for(vector<string>::iterator it= argNames.begin(); it != argNames.end(); ++it)
			{
				if (argTable->find(*it) != argTable->end())
					throw Error("identifier already declared",t);
				if (isVar)
					argTable->insert(pair<string, SymVar*> (*it, new SymVarParam_var(*it, Type)));
				else
					argTable->insert(pair<string, SymVar*> (*it, new SymVarParam(*it, Type)));
			}
			isVar = false;
			argNames.clear();
			if (t.GetValue() != ";" && t.GetValue() != ")")
				throw Error("incorrect function defenition", t);
		}
	}
	while(t.GetValue() != ")");
	t = sc.GetNextToken();
	return argTable;
}

void MainWindow::on_selectFile_clicked()
{
  QFileDialog dialog;
  if (dialog.exec())
  {
    if (!dialog.selectedFiles().size()) return;
    int counter = 0;
    QFile file(dialog.selectedFiles()[0]);
    file.open(QFile::ReadOnly);
    Tokenizer tokenizer(file);
    Token tok;
    QString sb;
    QString sb1k;
    QString sb32k;
    do
    {
      tok = tokenizer.Next();
      if (tok.GetType() == Token::eof) break;
      if (!(counter++ % 256))
      {
        std::cout << "counter: " << counter << std::endl;
      }
      sb1k = sb1k % tok.GetText();
      if (sb1k.size() == 256)
      {
        sb32k = sb32k % sb1k;
        sb1k.clear();
        if (sb32k.size() == 32768)
        {
          sb = sb % sb32k;
          sb32k.clear();
        }
      }
    } while (true);
    if (sb1k.size())
    {
      sb32k = sb32k % sb1k;
    }
    if (sb32k.size())
    {
      sb = sb % sb32k;
    }
    ui->textBrowser->setText(sb);
    file.close();
  }
}

Token &
	Compiler::
	Parse()
{
	Token *
		ret;
	bool
		real = false;
	for ( ; ; )
	{
		ret = &m_parser.Statement();
		// We need to make sure this
		// is really the end of the
		// stream.  When a new file
		// is added after all the
		// others have been done, then
		// end of stream token is
		// still in the system, even
		// though it's no longer the
		// end of the stream.
		TokenType::E_TYPE
			type = ret->GetType();
		if (type == TokenType::TT__END)
		{
			if (real)
			{
				// Really is the end
				break;
			}
			else
			{
				real = true;
			}
		}
		else if (type != TokenType::TT__EOF)
		{
			break;
		}
	}
	return *ret;
}

NodeStatement* Parser::ParseElement()
{
	if (t.GetValue() == "begin")
		return ParseBlock();
	if (t.GetValue() == "if") 
		return ParseIf();
	if (t.GetValue() == "while")
		return ParseWhile();
	if (t.GetValue() == "repeat")
		return ParseRepeat();
	if (t.GetValue() == "for")
		return ParseFor();
	if (t.GetValue() == "write")
		return ParseWrite(true, false);
	if (t.GetValue() == "writeln")
		return ParseWrite(true, true);
	if (t.GetValue() == "exit")
	{
		t = sc.GetNextToken();
		return new StatementExit(true);
	}
	if (t.GetType() == identifier)
	{
		bool found = false;
		_Table::iterator it = TableStack.Find(t.GetValue(), found);
		if (found)
		{
			if (it->second->IsProc() || it->second->IsFunc())
			{
				t = sc.GetNextToken();
				return ParseSub((SymProc*)it->second);
			}
		}
		return ParseAssignment();
	}
	if (t.GetValue() == "(")
	{
		return ParseAssignment();
	}
	throw Error("Statement error", t);
}

Statement *Statement::GetNext(Tokenizer &program) {
  Token token = program.Look();
  if (token.GetType() == "UNKNOWN")
    MalformedExpressionError::Raise(token, __FILE__, __LINE__);

  Statement *statement;
  if (token.Match("INTEGER_TYPE") || token.Match("STRING_TYPE"))
    statement = new AssignStatement();
  else if (token.Match("OUTPUT"))
    statement = new OutputStatement();
  else if (token.Match("IF"))
    statement = new IfStatement();
  else if (token.Match("SPAWN"))
    statement = new SpawnStatement();
  else if (token.Match("READ_LINE"))
    statement = new ReadLineStatement();

  statement->Read(program);

  LOG(INFO) << (std::string) typeid(*statement).name() << " load";
  return statement;
}

sint32 FilenameDB::Parse(char *filename)

{

    Token *token = new Token(filename, C3DIR_GAMEDATA);
	Assert(token);

   	if (token->GetType() != TOKEN_NUMBER) {
		c3errors_ErrorDialog  (token->ErrStr(), "Missing number of filenames");
        g_abort_parse = TRUE;
		delete token;
		return FALSE;
	} else {
        sint32 n;
		token->GetNumber(n);
		token->Next();
		if (n <0) {
			c3errors_ErrorDialog(token->ErrStr(), "Number of filename is negative");
            g_abort_parse = TRUE;
			delete token;
			return FALSE;
		}
		SetSize(n);
	}

    int count = 0;
    while (ParseAFilename(token, count)) {
        count++;
    }
    if (g_abort_parse) {
		delete token;
		return FALSE;
	}

	delete token;

    return TRUE;
}

SymProc* Parser::ProcFunc(bool isFunc)
{
	t = sc.GetNextToken();
	SymType* Type = NULL;
	SymProc* sub = NULL;
	if (t.GetType() != identifier)
		throw Error("incorrect procedure name", t);
	SymTable* argTable = NULL;
	vector<string>* argNames = new vector<string>;
	_Table::iterator it = table->find(t.GetValue());
	string name = t.GetValue();
	t = sc.GetNextToken();
	argTable = ParseArguments(argNames);
	TableStack.PushTable(argTable);
	if (isFunc)
	{
		if (t.GetValue() != ":")
			throw Error("type expected", t);
		Type = ParseType(false);
		t = sc.GetNextToken();
	}
	t = RequireToken(";", "\";\" expected");
	if (it != table->end())
	{
		if (!(it->second->IsProc()) || !(((SymProc*)it->second)->IsForward()))
			throw Error("incorrect procedure definition", t);
		SymTable* argTableF = ((SymProc*)it->second)->GetArgTable();
		vector<string>* argNamesF = ((SymProc*)it->second)->GetArgNames();
		for (size_t i=0; i<argNames->size(); ++i)
		{
			if ((*argNames)[i] != (*argNamesF)[i] ||
				!SymComp(argTable->find((*argNames)[i])->second , argTableF->find((*argNamesF)[i])->second))
				throw Error("Header does not match previouse definition", t);
		}
		if (isFunc)
			sub = ParseProcDecl(name, argNames, argTable, true, Type);
		else
			sub = ParseProcDecl(name, argNames, argTable, false, Type);
		t = RequireToken(";", "\";\" expected");
	}
	else
	{
		if (t.GetValue() == "forward")
		{
			t = sc.GetNextToken();
			if (t.GetValue() != ";")
				throw Error("\";\" expected", t);
			if (isFunc)
			{
				sub = new SymFunc(name, argNames, argTable, NULL, true, NULL);
				((SymFunc*)sub)->SetType(Type);
			}
			else
				sub = new SymProc(name, argNames, argTable, NULL, true);
			sub->print(os, false);
			table->insert(pair<string, SymProc*> (name, sub));
			TableStack.PopTable();
		}
		else
		{
			sub = ParseProcDecl(name, argNames, argTable, isFunc, Type);
			if (isFunc)
				((SymFunc*)sub)->SetType(Type);
			sub->GetBody()->print(os, 0);
		}
		t = sc.GetNextToken();
	}
	return sub;
}