C++ Tokenizer::peek方法代码示例

// ----------------------------------------------------------------------------
// ParseTreeNode::parse
//
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// ----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
	// Keep parsing until final } is reached (or end of file)
	string name, type;
	while (!tz.atEnd() && tz.current() != '}')
	{
		// Check for preprocessor stuff
		if (parser_ && tz.current()[0] == '#')
		{
			if (!parsePreprocessor(tz))
				return false;

			tz.advToNextLine();
			continue;
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(tz.current().text[0]))
		{
			logError(tz, S_FMT("Unexpected special character '%s'", CHR(tz.current().text)));
			return false;
		}

		// So we have either a node or property name
		name = tz.current().text;
		type.Empty();
		if (name.empty())
		{
			logError(tz, "Unexpected empty string");
			return false;
		}

		// Check for type+name pair
		if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':')
		{
			type = name;
			name = tz.next().text;

			if (name.empty())
			{
				logError(tz, "Unexpected empty string");
				return false;
			}
		}

		//Log::debug(S_FMT("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text)));

		// Assignment
		if (tz.advIfNext('=', 2))
		{
			if (!parseAssignment(tz, addChildPTN(name, type)))
				return false;
		}

		// Child node
		else if (tz.advIfNext('{', 2))
		{
			// Parse child node
			if (!addChildPTN(name, type)->parse(tz))
				return false;
		}

		// Child node (with no values/children)
		else if (tz.advIfNext(';', 2))
		{
			// Add child node
			addChildPTN(name, type);
			continue;
		}

		// Child node + inheritance
		else if (tz.advIfNext(':', 2))
		{
			// Check for opening brace
			if (tz.checkNext('{'))
			{
				// Add child node
				auto child = addChildPTN(name, type);
				child->inherit_ = tz.current().text;

				// Skip {
				tz.adv(2);

				// Parse child node
				if (!child->parse(tz))
					return false;
			}
//.........这里部分代码省略.........

// ----------------------------------------------------------------------------
// ParseTreeNode::parseAssignment
//
// Parses an assignment operation at [tz]'s current token to [child]
// ----------------------------------------------------------------------------
bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const
{
	// Check type of assignment list
	char list_end = ';';
	if (tz.current() == '{' && !tz.current().quoted_string)
	{
		list_end = '}';
		tz.adv();
	}

	// Parse until ; or }
	while (true)
	{
		auto& token = tz.current();

		// Check for list end
		if (token == list_end && !token.quoted_string)
			break;

		// Setup value
		Property value;

		// Detect value type
		if (token.quoted_string)	// Quoted string
			value = token.text;
		else if (token == "true")	// Boolean (true)
			value = true;
		else if (token == "false")	// Boolean (false)
			value = false;
		else if (token.isInteger())	// Integer
		{
			long val;
			token.text.ToLong(&val);
			value = (int)val;
		}
		else if (token.isHex())  	// Hex (0xXXXXXX)
		{
			long val;
			token.text.ToLong(&val, 0);
			value = (int)val;
		}
		else if (token.isFloat())	// Floating point
		{
			double val;
			token.text.ToDouble(&val);
			value = val;
		}
		else						// Unknown, just treat as string
			value = token.text;

		// Add value
		child->values_.push_back(value);

		// Check for ,
		if (tz.peek() == ',')
			tz.adv();	// Skip it
		else if (tz.peek() != list_end)
		{
			logError(
				tz,
				S_FMT("Expected \",\" or \"%c\", got \"%s\"", list_end, CHR(tz.peek().text))
			);
			return false;
		}

		tz.adv();
	}

	return true;
}

// -----------------------------------------------------------------------------
// Parses an assignment operation at [tz]'s current token to [child]
// -----------------------------------------------------------------------------
bool ParseTreeNode::parseAssignment(Tokenizer& tz, ParseTreeNode* child) const
{
	// Check type of assignment list
	char list_end = ';';
	if (tz.current() == '{' && !tz.current().quoted_string)
	{
		list_end = '}';
		tz.adv();
	}

	// Parse until ; or }
	while (true)
	{
		auto& token = tz.current();

		// Check for list end
		if (token == list_end && !token.quoted_string)
			break;

		// Setup value
		Property value;

		// Detect value type
		if (token.quoted_string) // Quoted string
			value = token.text;
		else if (token == "true") // Boolean (true)
			value = true;
		else if (token == "false") // Boolean (false)
			value = false;
		else if (token.isInteger()) // Integer
			value = token.asInt();
		else if (token.isHex()) // Hex (0xXXXXXX)
			value = token.asInt();
		else if (token.isFloat()) // Floating point
			value = token.asFloat();
		else // Unknown, just treat as string
			value = token.text;

		// Add value
		child->values_.push_back(value);

		// Check for ,
		if (tz.peek() == ',')
			tz.adv(); // Skip it
		else if (tz.peek() != list_end)
		{
			logError(tz, fmt::format(R"(Expected "," or "{}", got "{}")", list_end, tz.peek().text));
			return false;
		}

		tz.adv();
	}

	return true;
}

// -----------------------------------------------------------------------------
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// -----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
	// Keep parsing until final } is reached (or end of file)
	string name, type;
	while (!tz.atEnd() && tz.current() != '}')
	{
		// Check for preprocessor stuff
		if (parser_ && tz.current()[0] == '#')
		{
			if (!parsePreprocessor(tz))
				return false;

			tz.advToNextLine();
			continue;
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(tz.current().text[0]))
		{
			logError(tz, fmt::format("Unexpected special character '{}'", tz.current().text));
			return false;
		}

		// So we have either a node or property name
		name = tz.current().text;
		type.clear();
		if (name.empty())
		{
			logError(tz, "Unexpected empty string");
			return false;
//.........这里部分代码省略.........

// -----------------------------------------------------------------------------
// Reads and parses the SLADE configuration file
// -----------------------------------------------------------------------------
void readConfigFile()
{
	// Open SLADE.cfg
	Tokenizer tz;
	if (!tz.openFile(App::path("slade3.cfg", App::Dir::User)))
		return;

	// Go through the file with the tokenizer
	while (!tz.atEnd())
	{
		// If we come across a 'cvars' token, read in the cvars section
		if (tz.advIf("cvars", 2))
		{
			// Keep reading name/value pairs until we hit the ending '}'
			while (!tz.checkOrEnd("}"))
			{
				CVar::set(tz.current().text, tz.peek().text);
				tz.adv(2);
			}

			tz.adv(); // Skip ending }
		}

		// Read base resource archive paths
		if (tz.advIf("base_resource_paths", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				archive_manager.addBaseResourcePath(tz.current().text);
				tz.adv();
			}

			tz.adv(); // Skip ending }
		}

		// Read recent files list
		if (tz.advIf("recent_files", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				archive_manager.addRecentFile(tz.current().text);
				tz.adv();
			}

			tz.adv(); // Skip ending }
		}

		// Read keybinds
		if (tz.advIf("keys", 2))
			KeyBind::readBinds(tz);

		// Read nodebuilder paths
		if (tz.advIf("nodebuilder_paths", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				NodeBuilders::addBuilderPath(tz.current().text, tz.peek().text);
				tz.adv(2);
			}

			tz.adv(); // Skip ending }
		}

		// Read game exe paths
		if (tz.advIf("executable_paths", 2))
		{
			while (!tz.checkOrEnd("}"))
			{
				Executables::setGameExePath(tz.current().text, tz.peek().text);
				tz.adv(2);
			}

			tz.adv(); // Skip ending }
		}

		// Read window size/position info
		if (tz.advIf("window_info", 2))
			Misc::readWindowInfo(tz);

		// Next token
		tz.adv();
	}
}

// -----------------------------------------------------------------------------
// Parses a ZScript 'statement'. This isn't technically correct but suits our
// purposes well enough
//
// tokens
// {
//     block[0].tokens
//     {
//         block[0].block[0].tokens;
//         ...
//     }
//
//     block[1].tokens;
//     ...
// }
// -----------------------------------------------------------------------------
bool ParsedStatement::parse(Tokenizer& tz)
{
	// Check for unexpected token
	if (tz.check('}'))
	{
		tz.adv();
		return false;
	}

	line = tz.lineNo();

	// Tokens
	bool in_initializer = false;
	while (true)
	{
		// End of statement (;)
		if (tz.advIf(';'))
			return true;

		// DB comment
		if (tz.current().text.StartsWith(db_comment))
		{
			tokens.push_back(tz.current().text);
			tokens.push_back(tz.getLine());
			return true;
		}

		if (tz.check('}'))
		{
			// End of array initializer
			if (in_initializer)
			{
				in_initializer = false;
				tokens.emplace_back("}");
				tz.adv();
				continue;
			}

			// End of statement
			return true;
		}

		if (tz.atEnd())
		{
			Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line));
			return false;
		}

		// Beginning of block
		if (tz.advIf('{'))
			break;

		// Array initializer: ... = { ... }
		if (tz.current().text.Cmp("=") == 0 && tz.peek() == '{')
		{
			tokens.emplace_back("=");
			tokens.emplace_back("{");
			tz.adv(2);
			in_initializer = true;
			continue;
		}

		tokens.push_back(tz.current().text);
		tz.adv();
	}

	// Block
	while (true)
	{
		if (tz.advIf('}'))
			return true;

		if (tz.atEnd())
		{
			Log::debug(S_FMT("Failed parsing zscript statement/block beginning line %u", line));
			return false;
		}

		block.push_back({});
		block.back().entry = entry;
		if (!block.back().parse(tz) || block.back().tokens.empty())
			block.pop_back();
	}
}