C++ Tokenizer::lineNo方法代码示例

/* TextEditor::openJumpToDialog
 * Initialises and opens the 'Jump To' dialog
 *******************************************************************/
void TextEditor::openJumpToDialog()
{
	// Can't do this without a language definition or defined blocks
	if (!language || language->nJumpBlocks() == 0)
		return;

	// --- Scan for functions/scripts ---
	Tokenizer tz;
	vector<jp_t> jump_points;
	tz.openString(GetText());

	string token = tz.getToken();
	while (!token.IsEmpty())
	{
		if (token == "{")
		{
			// Skip block
			while (!token.IsEmpty() && token != "}")
				token = tz.getToken();
		}

		for (unsigned a = 0; a < language->nJumpBlocks(); a++)
		{
			// Get jump block keyword
			string block = language->jumpBlock(a);
			long skip = 0;
			if (block.Contains(":"))
			{
				wxArrayString sp = wxSplit(block, ':');
				sp.back().ToLong(&skip);
				block = sp[0];
			}

			if (S_CMPNOCASE(token, block))
			{
				string name = tz.getToken();
				for (int s = 0; s < skip; s++)
					name = tz.getToken();

				for (unsigned i = 0; i < language->nJBIgnore(); ++i)
					if (S_CMPNOCASE(name, language->jBIgnore(i)))
						name = tz.getToken();

				// Numbered block, add block name
				if (name.IsNumber())
					name = S_FMT("%s %s", language->jumpBlock(a), name);
				// Unnamed block, use block name
				if (name == "{" || name == ";")
					name = language->jumpBlock(a);

				// Create jump point
				jp_t jp;
				jp.name = name;
				jp.line = tz.lineNo() - 1;
				jump_points.push_back(jp);
			}
		}

		token = tz.getToken();
	}

	// Do nothing if no jump points
	if (jump_points.size() == 0)
		return;


	// --- Setup/show dialog ---
	wxDialog dlg(this, -1, "Jump To...");
	wxBoxSizer* sizer = new wxBoxSizer(wxVERTICAL);
	dlg.SetSizer(sizer);

	// Add Jump to dropdown
	wxChoice* choice_jump_to = new wxChoice(&dlg, -1);
	sizer->Add(choice_jump_to, 0, wxEXPAND|wxALL, 4);
	for (unsigned a = 0; a < jump_points.size(); a++)
		choice_jump_to->Append(jump_points[a].name);
	choice_jump_to->SetSelection(0);

	// Add dialog buttons
	sizer->Add(dlg.CreateButtonSizer(wxOK|wxCANCEL), 0, wxEXPAND|wxLEFT|wxRIGHT|wxBOTTOM, 4);

	// Show dialog
	dlg.SetInitialSize(wxSize(250, -1));
	dlg.CenterOnParent();
	if (dlg.ShowModal() == wxID_OK)
	{
		int selection = choice_jump_to->GetSelection();
		if (selection >= 0 && selection < (int)jump_points.size())
		{
			// Get line number
			int line = jump_points[selection].line;

			// Move to line
			int pos = GetLineEndPosition(line);
			SetCurrentPos(pos);
			SetSelection(pos, pos);
			SetFirstVisibleLine(line);
//.........这里部分代码省略.........

/* ParseTreeNode::parse
 * Parses formatted text data. Current valid formatting is:
 * (type) child = value;
 * (type) child = value1, value2, ...;
 * (type) child = { value1, value2, ... }
 * (type) child { grandchild = value; etc... }
 * (type) child : inherited { ... }
 * All values are read as strings, but can be retrieved as string,
 * int, bool or float.
 *******************************************************************/
bool ParseTreeNode::parse(Tokenizer& tz)
{
    // Get first token
    string token = tz.getToken();

    // Keep parsing until final } is reached (or end of file)
    while (!(S_CMP(token, "}")) && !token.IsEmpty())
    {
        // Check for preprocessor stuff
        if (parser)
        {
            // #define
            if (S_CMPNOCASE(token, "#define"))
            {
                parser->define(tz.getToken());
                token = tz.getToken();
                continue;
            }

            // #if(n)def
            if (S_CMPNOCASE(token, "#ifdef") || S_CMPNOCASE(token, "#ifndef"))
            {
                bool test = true;
                if (S_CMPNOCASE(token, "#ifndef"))
                    test = false;
                string define = tz.getToken();
                if (parser->defined(define) == test)
                {
                    // Continue
                    token = tz.getToken();
                    continue;
                }
                else
                {
                    // Skip section
                    int skip = 0;
                    while (true)
                    {
                        token = tz.getToken();
                        if (S_CMPNOCASE(token, "#endif"))
                            skip--;
                        else if (S_CMPNOCASE(token, "#ifdef"))
                            skip++;
                        else if (S_CMPNOCASE(token, "#ifndef"))
                            skip++;

                        if (skip < 0)
                            break;
                        if (token.IsEmpty())
                        {
                            wxLogMessage("Error: found end of file within #if(n)def block");
                            break;
                        }
                    }

                    continue;
                }
            }

            // #include (ignore)
            if (S_CMPNOCASE(token, "#include"))
            {
                tz.skipToken();	// Skip include path
                token = tz.getToken();
                continue;
            }

            // #endif (ignore)
            if (S_CMPNOCASE(token, "#endif"))
            {
                token = tz.getToken();
                continue;
            }
        }

        // If it's a special character (ie not a valid name), parsing fails
        if (tz.isSpecialCharacter(token.at(0)))
        {
            wxLogMessage("Parsing error: Unexpected special character '%s' in %s (line %d)", CHR(token), CHR(tz.getName()), tz.lineNo());
            return false;
        }

        // So we have either a node or property name
        string name = token;

        // Check next token to determine what we're doing
        string next = tz.peekToken();

        // Check for type+name pair
        string type = "";
//.........这里部分代码省略.........

// -----------------------------------------------------------------------------
// Parses ZMAPINFO-format definitions in [entry]
// -----------------------------------------------------------------------------
bool MapInfo::parseZMapInfo(ArchiveEntry* entry)
{
	Tokenizer tz;
	tz.setReadLowerCase(true);
	tz.openMem(entry->data(), entry->name());

	while (!tz.atEnd())
	{
		// Include
		if (tz.check("include"))
		{
			// Get entry at include path
			auto include_entry = entry->parent()->entryAtPath(tz.next().text);

			if (!include_entry)
			{
				Log::warning(
					"Warning - Parsing ZMapInfo \"{}\": Unable to include \"{}\" at line {}",
					entry->name(),
					tz.current().text,
					tz.lineNo());
			}
			else if (!parseZMapInfo(include_entry))
				return false;
		}

		// Map
		else if (tz.check("map") || tz.check("defaultmap") || tz.check("adddefaultmap"))
		{
			if (!parseZMap(tz, tz.current().text))
				return false;
		}

		// DoomEdNums
		else if (tz.check("doomednums"))
		{
			if (!parseDoomEdNums(tz))
				return false;
		}

		// Unknown block (skip it)
		else if (tz.check("{"))
		{
			Log::warning(2, "Warning - Parsing ZMapInfo \"{}\": Skipping {{}} block", entry->name());

			tz.adv();
			tz.skipSection("{", "}");
			continue;
		}

		// Unknown
		else
		{
			Log::warning(2, R"(Warning - Parsing ZMapInfo "{}": Unknown token "{}")", entry->name(), tz.current().text);
		}

		tz.adv();
	}

	Log::info(2, "Parsed ZMapInfo entry {} successfully", entry->name());

	return true;
}

// -----------------------------------------------------------------------------
// Parses a ZMAPINFO map definition of [type] beginning at the current token in
// tokenizer [tz]
// -----------------------------------------------------------------------------
bool MapInfo::parseZMap(Tokenizer& tz, std::string_view type)
{
	// TODO: Handle adddefaultmap
	auto map = default_map_;

	// Normal map, get lump/name/etc
	tz.adv();
	if (type == "map")
	{
		// Entry name should be just after map keyword
		map.entry_name = tz.current().text;

		// Parse map name
		tz.adv();
		if (tz.check("lookup"))
		{
			map.lookup_name = true;
			map.name        = tz.next().text;
		}
		else
		{
			map.lookup_name = false;
			map.name        = tz.current().text;
		}

		tz.adv();
	}

	if (!tz.advIf("{"))
//.........这里部分代码省略.........

// -----------------------------------------------------------------------------
// Returns true if the next token in [tz] is '='. If not, logs an error message
// -----------------------------------------------------------------------------
bool MapInfo::checkEqualsToken(Tokenizer& tz, std::string_view parsing) const
{
	if (tz.next() != "=")
	{
		Log::error("Error Parsing {}: Expected \"=\", got \"{}\" at line {}", parsing, tz.current().text, tz.lineNo());
		return false;
	}

	return true;
}

/* JumpToCalculator::Entry
 * JumpToCalculator thread entry function
 *******************************************************************/
wxThread::ExitCode JumpToCalculator::Entry()
{
	string jump_points;

	Tokenizer tz;
	tz.setSpecialCharacters(";,:|={}/()");
	tz.openString(text);

	string token = tz.getToken();
	while (!tz.atEnd())
	{
		if (token == "{")
		{
			// Skip block
			while (!tz.atEnd() && token != "}")
				token = tz.getToken();
		}

		for (unsigned a = 0; a < block_names.size(); a++)
		{
			// Get jump block keyword
			string block = block_names[a];
			long skip = 0;
			if (block.Contains(":"))
			{
				wxArrayString sp = wxSplit(block, ':');
				sp.back().ToLong(&skip);
				block = sp[0];
			}

			if (S_CMPNOCASE(token, block))
			{
				string name = tz.getToken();
				for (int s = 0; s < skip; s++)
					name = tz.getToken();

				for (unsigned i = 0; i < ignore.size(); ++i)
					if (S_CMPNOCASE(name, ignore[i]))
						name = tz.getToken();

				// Numbered block, add block name
				if (name.IsNumber())
					name = S_FMT("%s %s", block, name);
				// Unnamed block, use block name
				if (name == "{" || name == ";")
					name = block;

				// Add jump point
				jump_points += S_FMT("%d,%s,", tz.lineNo() - 1, CHR(name));
			}
		}

		token = tz.getToken();
	}

	// Remove ending comma
	if (!jump_points.empty())
		jump_points.RemoveLast(1);

	// Send event
	wxThreadEvent* event = new wxThreadEvent(wxEVT_COMMAND_JTCALCULATOR_COMPLETED);
	event->SetString(jump_points);
	wxQueueEvent(handler, event);

	return nullptr;
}

// -----------------------------------------------------------------------------
// Parses a ZScript 'statement'. This isn't technically correct but suits our
// purposes well enough
//
// tokens
// {
//     block[0].tokens
//     {
//         block[0].block[0].tokens;
//         ...
//     }
//
//     block[1].tokens;
//     ...
// }
// -----------------------------------------------------------------------------
bool ParsedStatement::parse(Tokenizer& tz)
{
	// Check for unexpected token
	if (tz.check('}'))
	{
		tz.adv();
		return false;
	}

	line = tz.lineNo();

	// Tokens
	bool in_initializer = false;
	while (true)
	{
		// End of statement (;)
		if (tz.advIf(';'))
			return true;

		// DB comment
		if (tz.current().text.StartsWith(db_comment))
		{
			tokens.push_back(tz.current().text);
			tokens.push_back(tz.getLine());
			return true;
		}

		if (tz.check('}'))
		{
			// End of array initializer
			if (in_initializer)
			{
				in_initializer = false;
				tokens.emplace_back("}");
				tz.adv();
				continue;
			}

			// End of statement
			return true;
		}

		if (tz.atEnd())
		{
			Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line));
			return false;
		}

		// Beginning of block
		if (tz.advIf('{'))
			break;

		// Array initializer: ... = { ... }
		if (tz.current().text.Cmp("=") == 0 && tz.peek() == '{')
		{
			tokens.emplace_back("=");
			tokens.emplace_back("{");
			tz.adv(2);
			in_initializer = true;
			continue;
		}

		tokens.push_back(tz.current().text);
		tz.adv();
	}

	// Block
	while (true)
	{
		if (tz.advIf('}'))
			return true;

		if (tz.atEnd())
		{
			Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line));
			return false;
		}

		block.push_back({});
		block.back().entry = entry;
		if (!block.back().parse(tz) || block.back().tokens.empty())
			block.pop_back();
	}
}