C++ Tokenizer::openString方法代码示例

/* ZTextureEditorPanel::onTextTranslationEnter
 * Called when the enter key is pressed in the translation text box
 *******************************************************************/
void ZTextureEditorPanel::onTextTranslationEnter(wxCommandEvent& e)
{
	// Parse translation text line
	Tokenizer tz;
	tz.openString(text_translation->GetValue());
	Translation trans;

	string token = tz.getToken();
	while (!token.IsEmpty())
	{
		// Parse the translation component
		trans.parse(token);

		// Skip ,
		if (tz.peekToken() == ",")
			tz.getToken();

		// Next component
		token = tz.getToken();
	}

	// Copy updated translation to all selected patches
	wxArrayInt selection = list_patches->selectedItems();
	for (unsigned a = 0; a < selection.size(); a++)
	{
		CTPatchEx* patchx = (CTPatchEx*)tex_current->getPatch(selection[a]);
		patchx->getTranslation().copy(trans);
	}

	// Update UI
	updatePatchControls();
	tex_canvas->redraw(true);

	tex_modified = true;
}

/* SCallTip::loadArgSet
 * Opens and displays the arg set [set] from the current function
 *******************************************************************/
void SCallTip::loadArgSet(int set)
{
	args.clear();
	if (!function->getArgSet(set).empty())
	{
		Tokenizer tz;
		tz.setSpecialCharacters("[],");
		tz.openString(function->getArgSet(set));
		string token = tz.getToken();
		vector<string> tokens;
		while (true)
		{
			tokens.push_back(token);

			string next = tz.peekToken();
			if (next == "," || next == "")
			{
				addArg(tokens);
				tokens.clear();
				tz.getToken();
			}

			if (next == "")
				break;

			token = tz.getToken();
		}
	}

	updateSize();

	Update();
	Refresh();
}

/* SImage::applyTranslation
 * Applies a palette translation to the image
 *******************************************************************/
bool SImage::applyTranslation(string tr, Palette8bit* pal)
{
	// Some hardcoded translations from ZDoom
	if (!tr.CmpNoCase("\"doom0\""))			tr = "\"112:127=96:111\"";
	else if (!tr.CmpNoCase("\"doom1\""))	tr = "\"112:127=64:79\"";
	else if (!tr.CmpNoCase("\"doom2\""))	tr = "\"112:127=32:47\"";
	else if (!tr.CmpNoCase("\"doom3\""))	tr = "\"112:127=88:103\"";
	else if (!tr.CmpNoCase("\"doom4\""))	tr = "\"112:127=56:71\"";
	else if (!tr.CmpNoCase("\"doom5\""))	tr = "\"112:127=176:191\"";
	else if (!tr.CmpNoCase("\"doom6\""))	tr = "\"112:127=192:207\"";
	else if (!tr.CmpNoCase("\"heretic0\""))	tr = "\"225:240=114:129\"";
	else if (!tr.CmpNoCase("\"heretic1\""))	tr = "\"225:240=145:160\"";
	else if (!tr.CmpNoCase("\"heretic2\""))	tr = "\"225:240=190:205\"";
	else if (!tr.CmpNoCase("\"heretic3\""))	tr = "\"225:240=67:82\"";
	else if (!tr.CmpNoCase("\"heretic4\""))	tr = "\"225:240=9:24\"";
	else if (!tr.CmpNoCase("\"heretic5\""))	tr = "\"225:240=74:89\"";
	else if (!tr.CmpNoCase("\"heretic6\""))	tr = "\"225:240=150:165\"";
	else if (!tr.CmpNoCase("\"heretic7\""))	tr = "\"225:240=192:207\"";
	else if (!tr.CmpNoCase("\"heretic8\""))	tr = "\"225:240=95:110\"";
	else if (!tr.CmpNoCase("\"strife0\""))	tr = "\"32:63=0:31\", \"128:143=64:79\", \"241:246=224:229\", \"247:251=241:245\"";
	else if (!tr.CmpNoCase("\"strife1\""))	tr = "\"32:63=0:31\", \"128:143=176:191\"";
	else if (!tr.CmpNoCase("\"strife2\""))	tr = "\"32:47=208:223\", \"48:63=208:223\", \"128:143=16:31\"";
	else if (!tr.CmpNoCase("\"strife3\""))	tr = "\"32:47=208:223\", \"48:63=208:223\", \"128:143=48:63\"";
	else if (!tr.CmpNoCase("\"strife4\""))	tr = "\"32:63=0:31\", \"80:95=128:143\", \"128:143=80:95\", \"192:223=160:191\"";
	else if (!tr.CmpNoCase("\"strife5\""))	tr = "\"32:63=0:31\", \"80:95=16:31\", \"128:143=96:111\", \"192:223=32:63\"";
	else if (!tr.CmpNoCase("\"strife6\""))	tr = "\"32:63=0:31\", \"80:95=64:79\", \"128:143=144:159\", \"192=1\", \"193:223=1:31\"";
	// Some more from Eternity
	else if (!tr.CmpNoCase("\"tomato\""))	tr = "\"112:113=171:171\", \"114:114=172:172\", \"115:122=173:187\", \"123:124=188:189\", \"125:126=45:47\", \"127:127=1:1\"";
	else if (!tr.CmpNoCase("\"dirt\""))		tr = "\"112:117=128:133\", \"118:120=135:137\", \"121:123=139:143\", \"124:125=237:239\", \"126:127=1:2\"";
	else if (!tr.CmpNoCase("\"blue\""))		tr = "\"112:121=197:206\", \"122:127=240:245";
	else if (!tr.CmpNoCase("\"gold\""))		tr = "\"112:113=160:160\", \"114:119=161:166\", \"120:123=236:239\", \"124:125=1:2\", \"126:127=7:8\"";
	else if (!tr.CmpNoCase("\"sea\""))		tr = "\"112:112=91:91\", \"113:114=94:95\", \"115:122=152:159\", \"123:126=9:12\", \"127:127=8:8\"";
	else if (!tr.CmpNoCase("\"black\""))	tr = "\"112:112=101:101\", \"113:121=103:111\", \"122:125=5:8\", \"126:127=0:0\"";
	else if (!tr.CmpNoCase("\"purple\""))	tr = "\"112:113=4:4\", \"114:115=170:170\", \"116:125=250:254\", \"126:127=46:46\"";
	else if (!tr.CmpNoCase("\"vomit\""))	tr = "\"112:119=209:216\", \"120:121=218:220\", \"122:124=69:75\", \"125:127=237:239\"";
	else if (!tr.CmpNoCase("\"pink\""))		tr = "\"112:113=16:17\", \"114:117=19:25\", \"118:119=27:28\", \"120:124=30:38\", \"125:126=41:43\", \"127:127=46:46\"";
	else if (!tr.CmpNoCase("\"cream\""))	tr = "\"112:112=4:4\", \"113:118=48:63\", \"119:119=65:65\", \"120:124=68:76\", \"125:126=77:79\", \"127:127=1:1\"";
	else if (!tr.CmpNoCase("\"white\""))	tr = "\"112:112=4:4\", \"113:115=80:82\", \"116:117=84:86\", \"118:120=89:93\", \"121:127=96:108\"";

	Translation trans;
	trans.clear();
	Tokenizer tz;
	tz.openString(tr);
	string token = tz.getToken();
	while (!token.IsEmpty())
	{
		trans.parse(token);
		tz.getToken();
		token = tz.getToken();
	}
	return applyTranslation(&trans, pal);
}

bool Parser::parseText(string& text, string source)
{
    // Open the given text data
    Tokenizer tz;
    if (!tz.openString(text, 0, 0, source))
    {
        wxLogMessage("Unable to open text data for parsing");
        return false;
    }

    // Do parsing
    return pt_root->parse(tz);
}

bool Parser::parseText(string& text, string source)
{
	Tokenizer tz;

	// Open the given text data
	tz.setReadLowerCase(!case_sensitive_);
	if (!tz.openString(text, 0, 0, source))
	{
		LOG_MESSAGE(1, "Unable to open text data for parsing");
		return false;
	}

	// Do parsing
	return pt_root_->parse(tz);
}

bool Parser::parseText(string_view text, string_view source) const
{
	Tokenizer tz;

	// Open the given text data
	tz.setReadLowerCase(!case_sensitive_);
	if (!tz.openString(text, 0, 0, source))
	{
		Log::error("Unable to open text data for parsing");
		return false;
	}

	// Do parsing
	return pt_root_->parse(tz);
}

/* TextEditor::openJumpToDialog
 * Initialises and opens the 'Jump To' dialog
 *******************************************************************/
void TextEditor::openJumpToDialog()
{
	// Can't do this without a language definition or defined blocks
	if (!language || language->nJumpBlocks() == 0)
		return;

	// --- Scan for functions/scripts ---
	Tokenizer tz;
	vector<jp_t> jump_points;
	tz.openString(GetText());

	string token = tz.getToken();
	while (!token.IsEmpty())
	{
		if (token == "{")
		{
			// Skip block
			while (!token.IsEmpty() && token != "}")
				token = tz.getToken();
		}

		for (unsigned a = 0; a < language->nJumpBlocks(); a++)
		{
			// Get jump block keyword
			string block = language->jumpBlock(a);
			long skip = 0;
			if (block.Contains(":"))
			{
				wxArrayString sp = wxSplit(block, ':');
				sp.back().ToLong(&skip);
				block = sp[0];
			}

			if (S_CMPNOCASE(token, block))
			{
				string name = tz.getToken();
				for (int s = 0; s < skip; s++)
					name = tz.getToken();

				for (unsigned i = 0; i < language->nJBIgnore(); ++i)
					if (S_CMPNOCASE(name, language->jBIgnore(i)))
						name = tz.getToken();

				// Numbered block, add block name
				if (name.IsNumber())
					name = S_FMT("%s %s", language->jumpBlock(a), name);
				// Unnamed block, use block name
				if (name == "{" || name == ";")
					name = language->jumpBlock(a);

				// Create jump point
				jp_t jp;
				jp.name = name;
				jp.line = tz.lineNo() - 1;
				jump_points.push_back(jp);
			}
		}

		token = tz.getToken();
	}

	// Do nothing if no jump points
	if (jump_points.size() == 0)
		return;


	// --- Setup/show dialog ---
	wxDialog dlg(this, -1, "Jump To...");
	wxBoxSizer* sizer = new wxBoxSizer(wxVERTICAL);
	dlg.SetSizer(sizer);

	// Add Jump to dropdown
	wxChoice* choice_jump_to = new wxChoice(&dlg, -1);
	sizer->Add(choice_jump_to, 0, wxEXPAND|wxALL, 4);
	for (unsigned a = 0; a < jump_points.size(); a++)
		choice_jump_to->Append(jump_points[a].name);
	choice_jump_to->SetSelection(0);

	// Add dialog buttons
	sizer->Add(dlg.CreateButtonSizer(wxOK|wxCANCEL), 0, wxEXPAND|wxLEFT|wxRIGHT|wxBOTTOM, 4);

	// Show dialog
	dlg.SetInitialSize(wxSize(250, -1));
	dlg.CenterOnParent();
	if (dlg.ShowModal() == wxID_OK)
	{
		int selection = choice_jump_to->GetSelection();
		if (selection >= 0 && selection < (int)jump_points.size())
		{
			// Get line number
			int line = jump_points[selection].line;

			// Move to line
			int pos = GetLineEndPosition(line);
			SetCurrentPos(pos);
			SetSelection(pos, pos);
			SetFirstVisibleLine(line);
//.........这里部分代码省略.........

/* Translation::parse
 * Parses a text definition [def] (in zdoom format, detailed here:
 * http://zdoom.org/wiki/Translation)
 *******************************************************************/
void Translation::parse(string def)
{
	// Open definition string for processing w/tokenizer
	Tokenizer tz;
	tz.setSpecialCharacters("[]:%,=");
	tz.openString(def);

	//wxLogMessage("Parse translation \"%s\"", CHR(def));

	// Read original range
	uint8_t o_start, o_end;
	o_start = tz.getInteger();
	if (tz.peekToken() == "=") o_end = o_start;
	else if (!tz.checkToken(":")) return;
	else o_end = tz.getInteger();
	if (!tz.checkToken("=")) return;

	// Check for reverse origin range
	bool reverse = (o_start > o_end);

	// Type of translation depends on next token
	if (tz.peekToken() == "[")
	{
		// Colour translation
		rgba_t start, end;

		tz.getToken();	// Skip [

		// Read start colour
		start.r = tz.getInteger();
		if (!tz.checkToken(",")) return;
		start.g = tz.getInteger();
		if (!tz.checkToken(",")) return;
		start.b = tz.getInteger();

		if (!tz.checkToken("]")) return;
		if (!tz.checkToken(":")) return;
		if (!tz.checkToken("[")) return;

		// Read end colour
		end.r = tz.getInteger();
		if (!tz.checkToken(",")) return;
		end.g = tz.getInteger();
		if (!tz.checkToken(",")) return;
		end.b = tz.getInteger();

		if (!tz.checkToken("]")) return;

		// Add translation
		TransRangeColour* tr = new TransRangeColour();
		if (reverse)
		{
			tr->o_start = o_end;
			tr->o_end = o_start;
			tr->d_start.set(end);
			tr->d_end.set(start);
		}
		else
		{
			tr->o_start = o_start;
			tr->o_end = o_end;
			tr->d_start.set(start);
			tr->d_end.set(end);
		}
		translations.push_back(tr);

		//wxLogMessage("Added colour translation");
	}
	else if (tz.peekToken() == "%")
	{
		// Desat colour translation
		float sr, sg, sb, er, eg, eb;

		tz.getToken();	// Skip %
		if (!tz.checkToken("[")) return;

		// Read start colour
		sr = tz.getFloat();
		if (!tz.checkToken(",")) return;
		sg = tz.getFloat();
		if (!tz.checkToken(",")) return;
		sb = tz.getFloat();

		if (!tz.checkToken("]")) return;
		if (!tz.checkToken(":")) return;
		if (!tz.checkToken("[")) return;

		// Read end colour
		er = tz.getFloat();
		if (!tz.checkToken(",")) return;
		eg = tz.getFloat();
		if (!tz.checkToken(",")) return;
		eb = tz.getFloat();

		if (!tz.checkToken("]")) return;

//.........这里部分代码省略.........

/* Console::execute
 * Attempts to execute the command line given
 *******************************************************************/
void Console::execute(string command)
{
	::logMessage(1, "> %s", CHR(command));

	// Don't bother doing anything else with an empty command
	if (command.size() == 0)
		return;

	// Add the command to the log
	cmd_log.insert(cmd_log.begin(), command);

	// Tokenize the command string
	Tokenizer tz;
	tz.openString(command);

	// Get the command name
	string cmd_name = tz.getToken();

	// Get all args
	string arg = tz.getToken();
	vector<string> args;
	while (arg != "")
	{
		args.push_back(arg);
		arg = tz.getToken();
	}

	// Check that it is a valid command
	//for (size_t a = 0; a < commands.size(); a++)
	for (Command& command : commands)
	{
		// Found it, execute and return
		if (command.getName() == cmd_name)
		{
			command.execute(args);
			return;
		}
	}

	// Check if it is a cvar
	CVar* cvar = get_cvar(cmd_name);
	if (cvar)
	{
		// Arg(s) given, set cvar value
		if (args.size() > 0)
		{
			if ((cvar->flags & CVAR_LOCKED) > 0)
				logMessage(S_FMT("CVar \"%s\" can not be modified via the console", CHR(cmd_name)));
			else if (cvar->type == CVAR_BOOLEAN)
			{
				if (args[0] == "0" || args[0] == "false")
					*((CBoolCVar*)cvar) = false;
				else
					*((CBoolCVar*)cvar) = true;
			}
			else if (cvar->type == CVAR_INTEGER)
				*((CIntCVar*)cvar) = atoi(CHR(args[0]));
			else if (cvar->type == CVAR_FLOAT)
				*((CFloatCVar*)cvar) = (float)atof(CHR(args[0]));
			else if (cvar->type == CVAR_STRING)
				*((CStringCVar*)cvar) = args[0];
		}

		// Print cvar value
		string value = "";
		if (cvar->type == CVAR_BOOLEAN)
		{
			if (cvar->GetValue().Bool)
				value = "true";
			else
				value = "false";
		}
		else if (cvar->type == CVAR_INTEGER)
			value = S_FMT("%d", cvar->GetValue().Int);
		else if (cvar->type == CVAR_FLOAT)
			value = S_FMT("%1.4f", cvar->GetValue().Float);
		else
			value = ((CStringCVar*)cvar)->value;

		logMessage(S_FMT("\"%s\" = \"%s\"", CHR(cmd_name), CHR(value)));

		if (cmd_name == "log_verbosity")
			Global::log_verbosity = cvar->GetValue().Int;

		return;
	}

	// Toggle global debug mode
	if (cmd_name == "debug")
	{
		Global::debug = !Global::debug;
		if (Global::debug)
			logMessage("Debugging stuff enabled");
		else
			logMessage("Debugging stuff disabled");

		return;
//.........这里部分代码省略.........

// ----------------------------------------------------------------------------
// TLFunction::addContext
//
// Adds a [context] of the function
// ----------------------------------------------------------------------------
void TLFunction::addContext(
	const string& context,
	const string& args,
	const string& return_type,
	string        description,
	const string& deprecated_f)
{
	contexts_.push_back(Context{ context, {}, return_type, description, "", "", "" });
	auto& ctx = contexts_.back();

	// Parse args
	Tokenizer tz;
	tz.setSpecialCharacters("[],");
	tz.openString(args);

	vector<string> arg_tokens;
	while (true)
	{
		while (!tz.check(","))
		{
			arg_tokens.push_back(tz.current().text);
			if (tz.atEnd())
				break;
			tz.adv();
		}

		ctx.params.push_back({});
		ctx.params.back().parse(arg_tokens);
		arg_tokens.clear();

		if (tz.atEnd())
			break;

		tz.adv();
	}

	if (!deprecated_f.empty())
	{
		// Parse deprecated string
		tz.openString(deprecated_f);

		for (unsigned t = 0; t < 2; t++)
		{
			while (tz.check(","))
				tz.adv();

			bool is_replacement = true;
			for (unsigned c = 0; c < tz.current().text.size(); c++)
			{
				char chr = tz.current().text[c];
				if (isdigit(chr) || chr == '.')
				{
					is_replacement = false;
					break;
				}
			}

			if (is_replacement)
				ctx.deprecated_f = tz.current().text;
			else
				ctx.deprecated_v = tz.current().text;

			if (tz.atEnd())
				break;
			tz.adv();
		}
	}
}

/* JumpToCalculator::Entry
 * JumpToCalculator thread entry function
 *******************************************************************/
wxThread::ExitCode JumpToCalculator::Entry()
{
	string jump_points;

	Tokenizer tz;
	tz.setSpecialCharacters(";,:|={}/()");
	tz.openString(text);

	string token = tz.getToken();
	while (!tz.atEnd())
	{
		if (token == "{")
		{
			// Skip block
			while (!tz.atEnd() && token != "}")
				token = tz.getToken();
		}

		for (unsigned a = 0; a < block_names.size(); a++)
		{
			// Get jump block keyword
			string block = block_names[a];
			long skip = 0;
			if (block.Contains(":"))
			{
				wxArrayString sp = wxSplit(block, ':');
				sp.back().ToLong(&skip);
				block = sp[0];
			}

			if (S_CMPNOCASE(token, block))
			{
				string name = tz.getToken();
				for (int s = 0; s < skip; s++)
					name = tz.getToken();

				for (unsigned i = 0; i < ignore.size(); ++i)
					if (S_CMPNOCASE(name, ignore[i]))
						name = tz.getToken();

				// Numbered block, add block name
				if (name.IsNumber())
					name = S_FMT("%s %s", block, name);
				// Unnamed block, use block name
				if (name == "{" || name == ";")
					name = block;

				// Add jump point
				jump_points += S_FMT("%d,%s,", tz.lineNo() - 1, CHR(name));
			}
		}

		token = tz.getToken();
	}

	// Remove ending comma
	if (!jump_points.empty())
		jump_points.RemoveLast(1);

	// Send event
	wxThreadEvent* event = new wxThreadEvent(wxEVT_COMMAND_JTCALCULATOR_COMPLETED);
	event->SetString(jump_points);
	wxQueueEvent(handler, event);

	return nullptr;
}