C++ Tokenizer::adv方法代码示例

// ----------------------------------------------------------------------------
// TLFunction::addContext
//
// Adds a [context] of the function
// ----------------------------------------------------------------------------
void TLFunction::addContext(
	const string& context,
	const string& args,
	const string& return_type,
	string        description,
	const string& deprecated_f)
{
	contexts_.push_back(Context{ context, {}, return_type, description, "", "", "" });
	auto& ctx = contexts_.back();

	// Parse args
	Tokenizer tz;
	tz.setSpecialCharacters("[],");
	tz.openString(args);

	vector<string> arg_tokens;
	while (true)
	{
		while (!tz.check(","))
		{
			arg_tokens.push_back(tz.current().text);
			if (tz.atEnd())
				break;
			tz.adv();
		}

		ctx.params.push_back({});
		ctx.params.back().parse(arg_tokens);
		arg_tokens.clear();

		if (tz.atEnd())
			break;

		tz.adv();
	}

	if (!deprecated_f.empty())
	{
		// Parse deprecated string
		tz.openString(deprecated_f);

		for (unsigned t = 0; t < 2; t++)
		{
			while (tz.check(","))
				tz.adv();

			bool is_replacement = true;
			for (unsigned c = 0; c < tz.current().text.size(); c++)
			{
				char chr = tz.current().text[c];
				if (isdigit(chr) || chr == '.')
				{
					is_replacement = false;
					break;
				}
			}

			if (is_replacement)
				ctx.deprecated_f = tz.current().text;
			else
				ctx.deprecated_v = tz.current().text;

			if (tz.atEnd())
				break;
			tz.adv();
		}
	}
}

// ----------------------------------------------------------------------------
// ParseTreeNode::parse
//
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// ----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
	// Keep parsing until final } is reached (or end of file)
	string name, type;
	while (!tz.atEnd() && tz.current() != '}')
	{
		// Check for preprocessor stuff
		if (parser_ && tz.current()[0] == '#')
		{
			if (!parsePreprocessor(tz))
				return false;

			tz.advToNextLine();
			continue;
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(tz.current().text[0]))
		{
			logError(tz, S_FMT("Unexpected special character '%s'", CHR(tz.current().text)));
			return false;
		}

		// So we have either a node or property name
		name = tz.current().text;
		type.Empty();
		if (name.empty())
		{
			logError(tz, "Unexpected empty string");
			return false;
		}

		// Check for type+name pair
		if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':')
		{
			type = name;
			name = tz.next().text;

			if (name.empty())
			{
				logError(tz, "Unexpected empty string");
				return false;
			}
		}

		//Log::debug(S_FMT("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text)));

		// Assignment
		if (tz.advIfNext('=', 2))
		{
			if (!parseAssignment(tz, addChildPTN(name, type)))
				return false;
		}

		// Child node
		else if (tz.advIfNext('{', 2))
		{
			// Parse child node
			if (!addChildPTN(name, type)->parse(tz))
				return false;
		}

		// Child node (with no values/children)
		else if (tz.advIfNext(';', 2))
		{
			// Add child node
			addChildPTN(name, type);
			continue;
		}

		// Child node + inheritance
		else if (tz.advIfNext(':', 2))
		{
			// Check for opening brace
			if (tz.checkNext('{'))
			{
				// Add child node
				auto child = addChildPTN(name, type);
				child->inherit_ = tz.current().text;

				// Skip {
				tz.adv(2);

				// Parse child node
				if (!child->parse(tz))
					return false;
			}
//.........这里部分代码省略.........

/* MapSpecials::processACSScripts
 * Process 'OPEN' ACS scripts for various specials - sector colours,
 * slopes, etc.
 *******************************************************************/
void MapSpecials::processACSScripts(ArchiveEntry* entry)
{
	sector_colours.clear();
	sector_fadecolours.clear();

	if (!entry || entry->getSize() == 0)
		return;

	Tokenizer tz;
	tz.setSpecialCharacters(";,:|={}/()");
	tz.openMem(entry->getMCData(), "ACS Scripts");

	while (!tz.atEnd())
	{
		if (tz.checkNC("script"))
		{
			LOG_MESSAGE(3, "script found");

			tz.adv(2);	// Skip script #

			// Check for open script
			if (tz.checkNC("OPEN"))
			{
				LOG_MESSAGE(3, "script is OPEN");

				// Skip to opening brace
				while (!tz.check("{"))
					tz.adv();

				// Parse script
				while (!tz.checkOrEnd("}"))
				{
					// --- Sector_SetColor ---
					if (tz.checkNC("Sector_SetColor"))
					{
						// Get parameters
						auto parameters = tz.getTokensUntil(")");

						// Parse parameters
						long val;
						int tag = -1;
						int r = -1;
						int g = -1;
						int b = -1;
						for (unsigned a = 0; a < parameters.size(); a++)
						{
							if (parameters[a].text.ToLong(&val))
							{
								if (tag < 0)
									tag = val;
								else if (r < 0)
									r = val;
								else if (g < 0)
									g = val;
								else if (b < 0)
									b = val;
							}
						}

						// Check everything is set
						if (b < 0)
						{
							LOG_MESSAGE(2, "Invalid Sector_SetColor parameters");
						}
						else
						{
							sector_colour_t sc;
							sc.tag = tag;
							sc.colour.set(r, g, b, 255);
							LOG_MESSAGE(3, "Sector tag %d, colour %d,%d,%d", tag, r, g, b);
							sector_colours.push_back(sc);
						}
					}
					// --- Sector_SetFade ---
					else if (tz.checkNC("Sector_SetFade"))
					{
						// Get parameters
						auto parameters = tz.getTokensUntil(")");

						// Parse parameters
						long val;
						int tag = -1;
						int r = -1;
						int g = -1;
						int b = -1;
						for (unsigned a = 0; a < parameters.size(); a++)
						{
							if (parameters[a].text.ToLong(&val))
							{
								if (tag < 0)
									tag = val;
								else if (r < 0)
									r = val;
								else if (g < 0)
									g = val;
								else if (b < 0)
//.........这里部分代码省略.........

// ----------------------------------------------------------------------------
// ParseTreeNode::parsePreprocessor
//
// Parses a preprocessor directive at [tz]'s current token
// ----------------------------------------------------------------------------
bool ParseTreeNode::parsePreprocessor(Tokenizer& tz)
{
	//Log::debug(S_FMT("Preprocessor %s", CHR(tz.current().text)));

	// #define
	if (tz.current() == "#define")
		parser_->define(tz.next().text);

	// #if(n)def
	else if (tz.current() == "#ifdef" || tz.current() == "#ifndef")
	{
		// Continue if condition succeeds
		bool test = true;
		if (tz.current() == "#ifndef")
			test = false;
		string define = tz.next().text;
		if (parser_->defined(define) == test)
			return true;

		// Failed condition, skip section
		int skip = 0;
		while (true)
		{
			auto& token = tz.next();
			if (token == "#endif")
				skip--;
			else if (token == "#ifdef")
				skip++;
			else if (token == "#ifndef")
				skip++;
			// TODO: #else

			if (skip < 0)
				break;
		}
	}

	// #include
	else if (tz.current() == "#include")
	{
		// Include entry at the given path if we have an archive dir set
		if (archive_dir_)
		{
			// Get entry to include
			auto inc_path = tz.next().text;
			auto archive = archive_dir_->archive();
			auto inc_entry = archive->entryAtPath(archive_dir_->getPath() + inc_path);
			if (!inc_entry) // Try absolute path
				inc_entry = archive->entryAtPath(inc_path);

			//Log::debug(S_FMT("Include %s", CHR(inc_path)));

			if (inc_entry)
			{
				// Save the current dir and set it to the included entry's dir
				auto orig_dir = archive_dir_;
				archive_dir_ = inc_entry->getParentDir();

				// Parse text in the entry
				Tokenizer inc_tz;
				inc_tz.openMem(inc_entry->getMCData(), inc_entry->getName());
				bool ok = parse(inc_tz);

				// Reset dir and abort if parsing failed
				archive_dir_ = orig_dir;
				if (!ok)
					return false;
			}
			else
				logError(tz, S_FMT("Include entry %s not found", CHR(inc_path)));
		}
		else
			tz.adv();	// Skip include path
	}

	// #endif (ignore)
	else if (tz.current() == "#endif")
		return true;

	// TODO: #else

	// Unrecognised
	else
		logError(tz, S_FMT("Unrecognised preprocessor directive \"%s\"", CHR(tz.current().text)));

	return true;
}

// ----------------------------------------------------------------------------
// ParseTreeNode::parseAssignment
//
// Parses an assignment operation at [tz]'s current token to [child]
// ----------------------------------------------------------------------------
bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const
{
	// Check type of assignment list
	char list_end = ';';
	if (tz.current() == '{' && !tz.current().quoted_string)
	{
		list_end = '}';
		tz.adv();
	}

	// Parse until ; or }
	while (true)
	{
		auto& token = tz.current();

		// Check for list end
		if (token == list_end && !token.quoted_string)
			break;

		// Setup value
		Property value;

		// Detect value type
		if (token.quoted_string)	// Quoted string
			value = token.text;
		else if (token == "true")	// Boolean (true)
			value = true;
		else if (token == "false")	// Boolean (false)
			value = false;
		else if (token.isInteger())	// Integer
		{
			long val;
			token.text.ToLong(&val);
			value = (int)val;
		}
		else if (token.isHex())  	// Hex (0xXXXXXX)
		{
			long val;
			token.text.ToLong(&val, 0);
			value = (int)val;
		}
		else if (token.isFloat())	// Floating point
		{
			double val;
			token.text.ToDouble(&val);
			value = val;
		}
		else						// Unknown, just treat as string
			value = token.text;

		// Add value
		child->values_.push_back(value);

		// Check for ,
		if (tz.peek() == ',')
			tz.adv();	// Skip it
		else if (tz.peek() != list_end)
		{
			logError(
				tz,
				S_FMT("Expected \",\" or \"%c\", got \"%s\"", list_end, CHR(tz.peek().text))
			);
			return false;
		}

		tz.adv();
	}

	return true;
}

// -----------------------------------------------------------------------------
// Parses ZMAPINFO-format definitions in [entry]
// -----------------------------------------------------------------------------
bool MapInfo::parseZMapInfo(ArchiveEntry* entry)
{
	Tokenizer tz;
	tz.setReadLowerCase(true);
	tz.openMem(entry->data(), entry->name());

	while (!tz.atEnd())
	{
		// Include
		if (tz.check("include"))
		{
			// Get entry at include path
			auto include_entry = entry->parent()->entryAtPath(tz.next().text);

			if (!include_entry)
			{
				Log::warning(
					"Warning - Parsing ZMapInfo \"{}\": Unable to include \"{}\" at line {}",
					entry->name(),
					tz.current().text,
					tz.lineNo());
			}
			else if (!parseZMapInfo(include_entry))
				return false;
		}

		// Map
		else if (tz.check("map") || tz.check("defaultmap") || tz.check("adddefaultmap"))
		{
			if (!parseZMap(tz, tz.current().text))
				return false;
		}

		// DoomEdNums
		else if (tz.check("doomednums"))
		{
			if (!parseDoomEdNums(tz))
				return false;
		}

		// Unknown block (skip it)
		else if (tz.check("{"))
		{
			Log::warning(2, "Warning - Parsing ZMapInfo \"{}\": Skipping {{}} block", entry->name());

			tz.adv();
			tz.skipSection("{", "}");
			continue;
		}

		// Unknown
		else
		{
			Log::warning(2, R"(Warning - Parsing ZMapInfo "{}": Unknown token "{}")", entry->name(), tz.current().text);
		}

		tz.adv();
	}

	Log::info(2, "Parsed ZMapInfo entry {} successfully", entry->name());

	return true;
}

// -----------------------------------------------------------------------------
// Parses a ZMAPINFO map definition of [type] beginning at the current token in
// tokenizer [tz]
// -----------------------------------------------------------------------------
bool MapInfo::parseZMap(Tokenizer& tz, std::string_view type)
{
	// TODO: Handle adddefaultmap
	auto map = default_map_;

	// Normal map, get lump/name/etc
	tz.adv();
	if (type == "map")
	{
		// Entry name should be just after map keyword
		map.entry_name = tz.current().text;

		// Parse map name
		tz.adv();
		if (tz.check("lookup"))
		{
			map.lookup_name = true;
			map.name        = tz.next().text;
		}
		else
		{
			map.lookup_name = false;
			map.name        = tz.current().text;
		}

		tz.adv();
	}

	if (!tz.advIf("{"))
//.........这里部分代码省略.........

// -----------------------------------------------------------------------------
// Reads and parses the SLADE configuration file
// -----------------------------------------------------------------------------
void readConfigFile()
{
	// Open SLADE.cfg
	Tokenizer tz;
	if (!tz.openFile(App::path("slade3.cfg", App::Dir::User)))
		return;

	// Go through the file with the tokenizer
	while (!tz.atEnd())
	{
		// If we come across a 'cvars' token, read in the cvars section
		if (tz.advIf("cvars", 2))
		{
			// Keep reading name/value pairs until we hit the ending '}'
			while (!tz.checkOrEnd("}"))
			{
				CVar::set(tz.current().text, tz.peek().text);
				tz.adv(2);
			}

			tz.adv(); // Skip ending }
		}

		// Read base resource archive paths
		if (tz.advIf("base_resource_paths", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				archive_manager.addBaseResourcePath(tz.current().text);
				tz.adv();
			}

			tz.adv(); // Skip ending }
		}

		// Read recent files list
		if (tz.advIf("recent_files", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				archive_manager.addRecentFile(tz.current().text);
				tz.adv();
			}

			tz.adv(); // Skip ending }
		}

		// Read keybinds
		if (tz.advIf("keys", 2))
			KeyBind::readBinds(tz);

		// Read nodebuilder paths
		if (tz.advIf("nodebuilder_paths", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				NodeBuilders::addBuilderPath(tz.current().text, tz.peek().text);
				tz.adv(2);
			}

			tz.adv(); // Skip ending }
		}

		// Read game exe paths
		if (tz.advIf("executable_paths", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				Executables::setGameExePath(tz.current().text, tz.peek().text);
				tz.adv(2);
			}

			tz.adv(); // Skip ending }
		}

		// Read window size/position info
		if (tz.advIf("window_info", 2))
			Misc::readWindowInfo(tz);

		// Next token
		tz.adv();
	}
}

// -----------------------------------------------------------------------------
// Parses an assignment operation at [tz]'s current token to [child]
// -----------------------------------------------------------------------------
bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const
{
	// Check type of assignment list
	char list_end = ';';
	if (tz.current() == '{' && !tz.current().quoted_string)
	{
		list_end = '}';
		tz.adv();
	}

	// Parse until ; or }
	while (true)
	{
		auto& token = tz.current();

		// Check for list end
		if (token == list_end && !token.quoted_string)
			break;

		// Setup value
		Property value;

		// Detect value type
		if (token.quoted_string) // Quoted string
			value = token.text;
		else if (token == "true") // Boolean (true)
			value = true;
		else if (token == "false") // Boolean (false)
			value = false;
		else if (token.isInteger()) // Integer
			value = token.asInt();
		else if (token.isHex()) // Hex (0xXXXXXX)
			value = token.asInt();
		else if (token.isFloat()) // Floating point
			value = token.asFloat();
		else // Unknown, just treat as string
			value = token.text;

		// Add value
		child->values_.push_back(value);

		// Check for ,
		if (tz.peek() == ',')
			tz.adv(); // Skip it
		else if (tz.peek() != list_end)
		{
			logError(tz, fmt::format(R"(Expected "," or "{}", got "{}")", list_end, tz.peek().text));
			return false;
		}

		tz.adv();
	}

	return true;
}

// -----------------------------------------------------------------------------
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// -----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
	// Keep parsing until final } is reached (or end of file)
	string name, type;
	while (!tz.atEnd() && tz.current() != '}')
	{
		// Check for preprocessor stuff
		if (parser_ && tz.current()[0] == '#')
		{
			if (!parsePreprocessor(tz))
				return false;

			tz.advToNextLine();
			continue;
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(tz.current().text[0]))
		{
			logError(tz, fmt::format("Unexpected special character '{}'", tz.current().text));
			return false;
		}

		// So we have either a node or property name
		name = tz.current().text;
		type.clear();
		if (name.empty())
		{
			logError(tz, "Unexpected empty string");
			return false;
//.........这里部分代码省略.........

// -----------------------------------------------------------------------------
// Parses a ZScript 'statement'. This isn't technically correct but suits our
// purposes well enough
//
// tokens
// {
//     block[0].tokens
//     {
//         block[0].block[0].tokens;
//         ...
//     }
//
//     block[1].tokens;
//     ...
// }
// -----------------------------------------------------------------------------
bool ParsedStatement::parse(Tokenizer& tz)
{
	// Check for unexpected token
	if (tz.check('}'))
	{
		tz.adv();
		return false;
	}

	line = tz.lineNo();

	// Tokens
	bool in_initializer = false;
	while (true)
	{
		// End of statement (;)
		if (tz.advIf(';'))
			return true;

		// DB comment
		if (tz.current().text.StartsWith(db_comment))
		{
			tokens.push_back(tz.current().text);
			tokens.push_back(tz.getLine());
			return true;
		}

		if (tz.check('}'))
		{
			// End of array initializer
			if (in_initializer)
			{
				in_initializer = false;
				tokens.emplace_back("}");
				tz.adv();
				continue;
			}

			// End of statement
			return true;
		}

		if (tz.atEnd())
		{
			Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line));
			return false;
		}

		// Beginning of block
		if (tz.advIf('{'))
			break;

		// Array initializer: ... = { ... }
		if (tz.current().text.Cmp("=") == 0 && tz.peek() == '{')
		{
			tokens.emplace_back("=");
			tokens.emplace_back("{");
			tz.adv(2);
			in_initializer = true;
			continue;
		}

		tokens.push_back(tz.current().text);
		tz.adv();
	}

	// Block
	while (true)
	{
		if (tz.advIf('}'))
			return true;

		if (tz.atEnd())
		{
			Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line));
			return false;
		}

		block.push_back({});
		block.back().entry = entry;
		if (!block.back().parse(tz) || block.back().tokens.empty())
			block.pop_back();
	}
}

// -----------------------------------------------------------------------------
// Reads in a ZDoom-format TEXTURES entry.
// Returns true on success, false otherwise
// -----------------------------------------------------------------------------
bool TextureXList::readTEXTURESData(ArchiveEntry* textures)
{
	// Check for empty entry
	if (!textures)
	{
		Global::error = "Attempt to read texture data from NULL entry";
		return false;
	}
	if (textures->size() == 0)
	{
		txformat_ = Format::Textures;
		return true;
	}

	// Get text to parse
	Tokenizer tz;
	tz.openMem(textures->data(), textures->name());

	// Parsing gogo
	while (!tz.atEnd())
	{
		// Texture definition
		if (tz.checkNC("Texture"))
		{
			auto tex = std::make_unique<CTexture>();
			if (tex->parse(tz, "Texture"))
				addTexture(std::move(tex));
		}

		// Sprite definition
		else if (tz.checkNC("Sprite"))
		{
			auto tex = std::make_unique<CTexture>();
			if (tex->parse(tz, "Sprite"))
				addTexture(std::move(tex));
		}

		// Graphic definition
		else if (tz.checkNC("Graphic"))
		{
			auto tex = std::make_unique<CTexture>();
			if (tex->parse(tz, "Graphic"))
				addTexture(std::move(tex));
		}

		// WallTexture definition
		else if (tz.checkNC("WallTexture"))
		{
			auto tex = std::make_unique<CTexture>();
			if (tex->parse(tz, "WallTexture"))
				addTexture(std::move(tex));
		}

		// Flat definition
		else if (tz.checkNC("Flat"))
		{
			auto tex = std::make_unique<CTexture>();
			if (tex->parse(tz, "Flat"))
				addTexture(std::move(tex));
		}

		// Old HIRESTEX "Define"
		else if (tz.checkNC("Define"))
		{
			auto tex = std::make_unique<CTexture>();
			if (tex->parseDefine(tz))
				addTexture(std::move(tex));
		}

		tz.adv();
	}

	txformat_ = Format::Textures;

	return true;
}