C++ Tokenizer::Next方法代码示例

void ppPragma::HandleSTDC(Tokenizer &tk)
{
    const Token *token = tk.Next();
    if (token && token->IsIdentifier())
    {
        std::string name = token->GetId();
        const Token *tokenCmd = tk.Next();
        if (tokenCmd && tokenCmd->IsIdentifier())
        {
            const char *val = tokenCmd->GetId().c_str();
            bool on;
            bool valid = false;
            if (!strcmp(val, "ON"))
            {
                valid = true;
                on = true;
            }
            else if (!strcmp(val, "OFF"))
            {
                valid = true;
                on = false;
                
            }
            if (valid)
            {
                if (name == "FENV_ACCESS")
                    FenvAccess::Instance()->Add(on);
                else if (name == "CX_LIMITED_RANGE")
                    CXLimitedRange::Instance()->Add(on);
                else if (name == "FP_CONTRACT")
                    FPContract::Instance()->Add(on);
            }
        }
    }
}

void ppPragma::HandlePack(Tokenizer &tk)
{
    const Token *tok = tk.Next();
    if (tok && tok->GetKeyword() == openpa)
    {
        tok = tk.Next();
        int val = -1;
        if (tok)
            if (tok->IsNumeric())
            {
                val = tok->GetInteger();
            }
        tok = tk.Next();
        if (tok)
            if (tok->GetKeyword() == closepa)
            {
                if (val <= 0)
                {
                    Packing::Instance()->Remove();
                }
                else
                {
                    Packing::Instance()->Add(val);
                }
            }
    }
}

void ppPragma::HandleSR(Tokenizer &tk, bool startup)
{
    const Token *name = tk.Next();
    if (name && name->IsIdentifier())
    {
        std::string id = name->GetId();
        const Token *prio = tk.Next();
        if (prio && prio->IsNumeric() && !prio->IsFloating())
        {
            Startups::Instance()->Add(id, prio->GetInteger(), startup);
        }
    }
}

void ppPragma::HandleAlias(Tokenizer &tk)
{
    const Token *name = tk.Next();
    if (name && name->IsIdentifier())
    {
        std::string id = name->GetId();
        const Token *alias = tk.Next();
        if (alias && alias->GetKeyword() == eq)
        {
            alias = tk.Next();
            if (alias && alias->IsIdentifier())
            {
                Aliases::Instance()->Add(id, alias->GetId());
            }
        }
    }
}

static std::vector<Statement*> ParseStatements(const String& input, bool allowSpaces = false)
{
	String string = input;
	// Remove spaces since they should be ignored (i.e. variables should not be split)
	if (allowSpaces)
		string.erase(std::remove(string.begin(), string.end(), ' '), string.end());

	string += ";";

	Tokenizer tokenizer;
	tokenizer.AddEndChars(";");
	tokenizer.AddOperatorChars("<=>&|\\/");
	tokenizer.AddWhitespaceChars(" \n\r\t");
	tokenizer.SetString(string);

	Token* lastToken = nullptr;
	Token token;
	Statement* statement = nullptr;
	std::vector<Statement*> statements;
	while (tokenizer.Next(token))
	{
		if (lastToken == nullptr || lastToken->type == Token::Type::END_STATEMENT)
		{
			statement = new Statement();
			statements.push_back(statement);
		}
		lastToken = &token;
		switch (token.type)
		{
		case Token::Type::IDENTIFIER:
			statement->identifiers.push_back(token.token);
			break;
		case Token::Type::OPERATOR:
			statement->operators.push_back(ParseOperator(token.token));
			break;
		}
	}
	return statements;
}

//=================================================================================================
bool RunInstallScripts()
{
	Info("Reading install scripts.");
	WIN32_FIND_DATA data;
	HANDLE find = FindFirstFile(Format("%s/install/*.txt", g_system_dir.c_str()), &data);
	if(find == INVALID_HANDLE_VALUE)
		return true;

	vector<InstallScript> scripts;

	Tokenizer t;
	t.AddKeyword("install", 0);
	t.AddKeyword("version", 1);
	t.AddKeyword("remove", 2);

	do
	{
		int major, minor, patch;

		// read file to find version info
		try
		{
			if(t.FromFile(Format("%s/install/%s", g_system_dir.c_str(), data.cFileName)))
			{
				t.Next();
				if(t.MustGetKeywordId() == 2)
				{
					// old install script
					if(sscanf_s(data.cFileName, "%d.%d.%d.txt", &major, &minor, &patch) != 3)
					{
						if(sscanf_s(data.cFileName, "%d.%d.txt", &major, &minor) == 2)
							patch = 0;
						else
						{
							// unknown version
							major = 0;
							minor = 0;
							patch = 0;
						}
					}
				}
				else
				{
					t.AssertKeyword(0);
					t.Next();
					if(t.MustGetInt() != 1)
						t.Throw(Format("Unknown install script version '%d'.", t.MustGetInt()));
					t.Next();
					t.AssertKeyword(1);
					t.Next();
					major = t.MustGetInt();
					t.Next();
					minor = t.MustGetInt();
					t.Next();
					patch = t.MustGetInt();
				}

				InstallScript& s = Add1(scripts);
				s.filename = data.cFileName;
				s.version = (((major & 0xFF) << 16) | ((minor & 0xFF) << 8) | (patch & 0xFF));
			}
		}
		catch(const Tokenizer::Exception& e)
		{
			Warn("Unknown install script '%s': %s", data.cFileName, e.ToString());
		}
	} while(FindNextFile(find, &data));

	FindClose(find);

	if(scripts.empty())
		return true;

	std::sort(scripts.begin(), scripts.end());

	GetModuleFileName(nullptr, BUF, 256);
	char buf[512], buf2[512];
	char* filename;
	GetFullPathName(BUF, 512, buf, &filename);
	*filename = 0;
	DWORD len = strlen(buf);

	LocalString s, s2;

	for(vector<InstallScript>::iterator it = scripts.begin(), end = scripts.end(); it != end; ++it)
	{
		cstring path = Format("%s/install/%s", g_system_dir.c_str(), it->filename.c_str());

		try
		{
			if(!t.FromFile(path))
			{
				Error("Failed to load install script '%s'.", it->filename.c_str());
				continue;
			}
			Info("Using install script %s.", it->filename.c_str());

			t.Next();
			t.AssertKeyword();
//.........这里部分代码省略.........