C++ Tokenizer::next方法代码示例

	//Sets the interaction mode based on command sent from tablet
	//Echoes a confirmation message back to tablet
	void Terrain::handleMessages(){
		std::string msg;
		msg = server->parseData();
		ostringstream ss;
		ss << "Handling Message: " << msg << std::endl;
		if(!msg.empty()){
			OutputDebugString(ss.str().c_str());
		}

		//Tokenize message
		Tokenizer *tok = new Tokenizer(msg, ",");
		std::string token = tok->next();
		//Prepare confirmation reply
		ostringstream replyss;
		replyss << "CFM," << msg;
		std::string reply = replyss.str();

		//Selecting type of interaction
		if(token == "placepoi"){
			token = tok->next();
			poiCube->setText(token);
			setInteractionMode(POI_MODE);
			//server->sendData("placepoi\n");
			server->sendData(reply.c_str());
		}			
		else if(msg == "placepath\n"){
			setInteractionMode(ENDPOINT_DRAWING_MODE);
			//server->sendData("placepath\n");
			server->sendData(reply.c_str());
		}
		else if(msg == "drawpath\n"){
			setInteractionMode(DIRECT_DRAWING_MODE);
			//server->sendData("drawpath\n");
			server->sendData(reply.c_str());
		}	
		else if(msg == "placeroi\n"){
			setInteractionMode(ROI_MODE);
			//server->sendData("placeroi\n");
			server->sendData(reply.c_str());
		}	
		else if(msg == "placepoi\n"){
			setInteractionMode(POI_MODE);
			//server->sendData("placepoi\n");
			server->sendData(reply.c_str());
		}
		else if(msg == "filterheight\n"){
			setInteractionMode(HEIGHT_FILTERING_MODE);
			server->sendData(reply.c_str());
		}
		else if(msg == "selectheight\n"){
			setInteractionMode(HEIGHT_SELECT_MODE);
			//server->sendData("selectheight\n");
			server->sendData(reply.c_str());
		}
		
	}

void QSGShaderSourceBuilder::removeVersion()
{
    Tokenizer tok;
    const char *input = m_source.constData();
    tok.initialize(input);

    // First find #version beginning and end (if present)
    const char *versionStartPos = 0;
    const char *versionEndPos = 0;
    bool inSingleLineComment = false;
    bool inMultiLineComment = false;
    bool foundVersionStart = false;

    Tokenizer::Token lt = Tokenizer::Token_Unspecified;
    Tokenizer::Token t = tok.next();
    while (t != Tokenizer::Token_EOF) {
        // Handle comment blocks
        if (t == Tokenizer::Token_MultiLineCommentStart )
            inMultiLineComment = true;
        if (t == Tokenizer::Token_MultiLineCommentEnd)
            inMultiLineComment = false;
        if (t == Tokenizer::Token_SingleLineComment)
            inSingleLineComment = true;
        if (t == Tokenizer::Token_NewLine && inSingleLineComment && !inMultiLineComment)
            inSingleLineComment = false;

        // Have we found #version, #extension or void main()?
        if (t == Tokenizer::Token_Version && !inSingleLineComment && !inMultiLineComment) {
            versionStartPos = tok.pos - 1;
            foundVersionStart = true;
        } else if (foundVersionStart && t == Tokenizer::Token_NewLine) {
            versionEndPos = tok.pos;
            break;
        } else if (lt == Tokenizer::Token_Void && t == Tokenizer::Token_Identifier) {
            if (qstrncmp("main", tok.identifier, 4) == 0)
                break;
        }

        // Scan to next token
        lt = t;
        t = tok.next();
    }

    if (versionStartPos == 0)
        return;

    // Construct a new shader string, inserting the definition
    QByteArray newSource;
    newSource.reserve(m_source.size() - (versionEndPos - versionStartPos));
    newSource += QByteArray::fromRawData(input, versionStartPos - input);
    newSource += QByteArray::fromRawData(versionEndPos, m_source.size() - (versionEndPos - versionStartPos));

    m_source = newSource;
}

void Runtime::accept( Token::Code token )
{
  if ( expect( token ) ) {
    zdebug() << "ACCEPTED TOKEN" << token;
    tokenizer.next();
  }
}

void Memory::loadFile(string fileName){
	ifstream file(fileName.c_str(), std::ifstream::in);
	
	//check if file is ready for reading
	if(file.good()){
		string line;
		string tkn;
		Tokenizer tokenizer;
		int cursor = 0;

		//read all lines
		while(getline(file, line)){
			tokenizer.set(line);
			tkn = tokenizer.next();
			if(tkn == ".1000")
				cursor = 1000;
			else{
				MemVector[cursor] = atoi(tkn.c_str());
				cursor++;
			}
		}
		file.close();
	}else
		cout << "Could not open file " << fileName << endl;
}

// -----------------------------------------------------------------------------
// Returns true if the next token in [tz] is '='. If not, logs an error message
// -----------------------------------------------------------------------------
bool MapInfo::checkEqualsToken(Tokenizer& tz, std::string_view parsing) const
{
	if (tz.next() != "=")
	{
		Log::error("Error Parsing {}: Expected \"=\", got \"{}\" at line {}", parsing, tz.current().text, tz.lineNo());
		return false;
	}

	return true;
}

// -----------------------------------------------------------------------------
// Parses all statements/blocks in [entry], adding them to [parsed]
// -----------------------------------------------------------------------------
void parseBlocks(ArchiveEntry* entry, vector<ParsedStatement>& parsed)
{
	Tokenizer tz;
	tz.setSpecialCharacters(CHR(Tokenizer::DEFAULT_SPECIAL_CHARACTERS + "()+-[]&!?."));
	tz.enableDecorate(true);
	tz.setCommentTypes(Tokenizer::CommentTypes::CPPStyle | Tokenizer::CommentTypes::CStyle);
	tz.openMem(entry->getMCData(), "ZScript");

	// Log::info(2, S_FMT("Parsing ZScript entry \"%s\"", entry->getPath(true)));

	while (!tz.atEnd())
	{
		// Preprocessor
		if (tz.current().text.StartsWith("#"))
		{
			if (tz.checkNC("#include"))
			{
				auto inc_entry = entry->relativeEntry(tz.next().text);

				// Check #include path could be resolved
				if (!inc_entry)
				{
					Log::warning(S_FMT(
						"Warning parsing ZScript entry %s: "
						"Unable to find #included entry \"%s\" at line %u, skipping",
						CHR(entry->getName()),
						CHR(tz.current().text),
						tz.current().line_no));
				}
				else
					parseBlocks(inc_entry, parsed);
			}

			tz.advToNextLine();
			continue;
		}

		// Version
		else if (tz.checkNC("version"))
		{
			tz.advToNextLine();
			continue;
		}

		// ZScript
		parsed.push_back({});
		parsed.back().entry = entry;
		if (!parsed.back().parse(tz))
			parsed.pop_back();
	}

	// Set entry type
	if (etype_zscript && entry->getType() != etype_zscript)
		entry->setType(etype_zscript);
}

bool	Texture :: loadCubemapFromNames ( const char * fileNames )
{
	const char * seps = ",;";
	Tokenizer	 tok ( fileNames, seps );
	string		 names [6];
	
	for ( int i = 0; i < 6; i++ )
	{
		names [i] = tok.next ();
		
		if ( names [i].empty () )
			return false;
	}
	
	return loadCubemap ( names [0].c_str (), names [1].c_str (), names [2].c_str (), names [3].c_str (), 
	                     names [4].c_str (), names [5].c_str () );
}

void QSGShaderSourceBuilder::addDefinition(const QByteArray &definition)
{
    if (definition.isEmpty())
        return;

    Tokenizer tok;
    const char *input = m_source.constData();
    tok.initialize(input);

    // First find #version, #extension's and "void main() { ... "
    const char *versionPos = 0;
    const char *extensionPos = 0;
    bool inSingleLineComment = false;
    bool inMultiLineComment = false;
    bool foundVersionStart = false;
    bool foundExtensionStart = false;

    Tokenizer::Token lt = Tokenizer::Token_Unspecified;
    Tokenizer::Token t = tok.next();
    while (t != Tokenizer::Token_EOF) {
        // Handle comment blocks
        if (t == Tokenizer::Token_MultiLineCommentStart )
            inMultiLineComment = true;
        if (t == Tokenizer::Token_MultiLineCommentEnd)
            inMultiLineComment = false;
        if (t == Tokenizer::Token_SingleLineComment)
            inSingleLineComment = true;
        if (t == Tokenizer::Token_NewLine && inSingleLineComment && !inMultiLineComment)
            inSingleLineComment = false;

        // Have we found #version, #extension or void main()?
        if (t == Tokenizer::Token_Version && !inSingleLineComment && !inMultiLineComment)
            foundVersionStart = true;

        if (t == Tokenizer::Token_Extension && !inSingleLineComment && !inMultiLineComment)
            foundExtensionStart = true;

        if (foundVersionStart && t == Tokenizer::Token_NewLine) {
            versionPos = tok.pos;
            foundVersionStart = false;
        } else if (foundExtensionStart && t == Tokenizer::Token_NewLine) {
            extensionPos = tok.pos;
            foundExtensionStart = false;
        } else if (lt == Tokenizer::Token_Void && t == Tokenizer::Token_Identifier) {
            if (qstrncmp("main", tok.identifier, 4) == 0)
                break;
        }

        // Scan to next token
        lt = t;
        t = tok.next();
    }

    // Determine where to insert the definition.
    // If we found #extension directives, insert after last one,
    // else, if we found #version insert after #version
    // otherwise, insert at beginning.
    const char *insertionPos = extensionPos ? extensionPos : (versionPos ? versionPos : input);

    // Construct a new shader string, inserting the definition
    QByteArray newSource;
    newSource.reserve(m_source.size() + definition.size() + 9);
    newSource += QByteArray::fromRawData(input, insertionPos - input);
    newSource += QByteArrayLiteral("#define ") + definition + QByteArrayLiteral("\n");
    newSource += QByteArray::fromRawData(insertionPos, m_source.size() - (insertionPos - input));

    m_source = newSource;
}

// ----------------------------------------------------------------------------
// ParseTreeNode::parse
//
// Parses formatted text data. Current valid formatting is:
// (type) child = value;
// (type) child = value1, value2, ...;
// (type) child = { value1, value2, ... }
// (type) child { grandchild = value; etc... }
// (type) child : inherited { ... }
//
// All values are read as strings, but can be retrieved as string, int, bool
// or float.
// ----------------------------------------------------------------------------
bool ParseTreeNode::parse(Tokenizer& tz)
{
	// Keep parsing until final } is reached (or end of file)
	string name, type;
	while (!tz.atEnd() && tz.current() != '}')
	{
		// Check for preprocessor stuff
		if (parser_ && tz.current()[0] == '#')
		{
			if (!parsePreprocessor(tz))
				return false;

			tz.advToNextLine();
			continue;
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(tz.current().text[0]))
		{
			logError(tz, S_FMT("Unexpected special character '%s'", CHR(tz.current().text)));
			return false;
		}

		// So we have either a node or property name
		name = tz.current().text;
		type.Empty();
		if (name.empty())
		{
			logError(tz, "Unexpected empty string");
			return false;
		}

		// Check for type+name pair
		if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':')
		{
			type = name;
			name = tz.next().text;

			if (name.empty())
			{
				logError(tz, "Unexpected empty string");
				return false;
			}
		}

		//Log::debug(S_FMT("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text)));

		// Assignment
		if (tz.advIfNext('=', 2))
		{
			if (!parseAssignment(tz, addChildPTN(name, type)))
				return false;
		}

		// Child node
		else if (tz.advIfNext('{', 2))
		{
			// Parse child node
			if (!addChildPTN(name, type)->parse(tz))
				return false;
		}

		// Child node (with no values/children)
		else if (tz.advIfNext(';', 2))
		{
			// Add child node
			addChildPTN(name, type);
			continue;
		}

		// Child node + inheritance
		else if (tz.advIfNext(':', 2))
		{
			// Check for opening brace
			if (tz.checkNext('{'))
			{
				// Add child node
				auto child = addChildPTN(name, type);
				child->inherit_ = tz.current().text;

				// Skip {
				tz.adv(2);

				// Parse child node
				if (!child->parse(tz))
					return false;
			}
//.........这里部分代码省略.........

// ----------------------------------------------------------------------------
// ParseTreeNode::parsePreprocessor
//
// Parses a preprocessor directive at [tz]'s current token
// ----------------------------------------------------------------------------
bool ParseTreeNode::parsePreprocessor(Tokenizer& tz)
{
	//Log::debug(S_FMT("Preprocessor %s", CHR(tz.current().text)));

	// #define
	if (tz.current() == "#define")
		parser_->define(tz.next().text);

	// #if(n)def
	else if (tz.current() == "#ifdef" || tz.current() == "#ifndef")
	{
		// Continue if condition succeeds
		bool test = true;
		if (tz.current() == "#ifndef")
			test = false;
		string define = tz.next().text;
		if (parser_->defined(define) == test)
			return true;

		// Failed condition, skip section
		int skip = 0;
		while (true)
		{
			auto& token = tz.next();
			if (token == "#endif")
				skip--;
			else if (token == "#ifdef")
				skip++;
			else if (token == "#ifndef")
				skip++;
			// TODO: #else

			if (skip < 0)
				break;
		}
	}

	// #include
	else if (tz.current() == "#include")
	{
		// Include entry at the given path if we have an archive dir set
		if (archive_dir_)
		{
			// Get entry to include
			auto inc_path = tz.next().text;
			auto archive = archive_dir_->archive();
			auto inc_entry = archive->entryAtPath(archive_dir_->getPath() + inc_path);
			if (!inc_entry) // Try absolute path
				inc_entry = archive->entryAtPath(inc_path);

			//Log::debug(S_FMT("Include %s", CHR(inc_path)));

			if (inc_entry)
			{
				// Save the current dir and set it to the included entry's dir
				auto orig_dir = archive_dir_;
				archive_dir_ = inc_entry->getParentDir();

				// Parse text in the entry
				Tokenizer inc_tz;
				inc_tz.openMem(inc_entry->getMCData(), inc_entry->getName());
				bool ok = parse(inc_tz);

				// Reset dir and abort if parsing failed
				archive_dir_ = orig_dir;
				if (!ok)
					return false;
			}
			else
				logError(tz, S_FMT("Include entry %s not found", CHR(inc_path)));
		}
		else
			tz.adv();	// Skip include path
	}

	// #endif (ignore)
	else if (tz.current() == "#endif")
		return true;

	// TODO: #else

	// Unrecognised
	else
		logError(tz, S_FMT("Unrecognised preprocessor directive \"%s\"", CHR(tz.current().text)));

	return true;
}

 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */
#include "tokenizer.hpp"
#include <sstream>
#include "catch.hpp"

TEST_CASE( "Parsing Tokens", "[Tokenizer::parse]" ) {
	Tokenizer tok;
	std::stringstream test;
	test <<
		"test_project {" << std::endl <<
		"test=1," << std::endl <<
		"	 meep = 39" << std::endl <<
		"}" << std::endl;

	tok = Tokenizer(test);
	CHECK(tok.next() == "test_project");
	Token token = tok.getToken();
	CHECK_FALSE(token.getType() == ERROR);
	CHECK(token.getType() == PROJECT);
	CHECK(token.getValue() == "test_project");
}

// -----------------------------------------------------------------------------
// Parses ZMAPINFO-format definitions in [entry]
// -----------------------------------------------------------------------------
bool MapInfo::parseZMapInfo(ArchiveEntry* entry)
{
	Tokenizer tz;
	tz.setReadLowerCase(true);
	tz.openMem(entry->data(), entry->name());

	while (!tz.atEnd())
	{
		// Include
		if (tz.check("include"))
		{
			// Get entry at include path
			auto include_entry = entry->parent()->entryAtPath(tz.next().text);

			if (!include_entry)
			{
				Log::warning(
					"Warning - Parsing ZMapInfo \"{}\": Unable to include \"{}\" at line {}",
					entry->name(),
					tz.current().text,
					tz.lineNo());
			}
			else if (!parseZMapInfo(include_entry))
				return false;
		}

		// Map
		else if (tz.check("map") || tz.check("defaultmap") || tz.check("adddefaultmap"))
		{
			if (!parseZMap(tz, tz.current().text))
				return false;
		}

		// DoomEdNums
		else if (tz.check("doomednums"))
		{
			if (!parseDoomEdNums(tz))
				return false;
		}

		// Unknown block (skip it)
		else if (tz.check("{"))
		{
			Log::warning(2, "Warning - Parsing ZMapInfo \"{}\": Skipping {{}} block", entry->name());

			tz.adv();
			tz.skipSection("{", "}");
			continue;
		}

		// Unknown
		else
		{
			Log::warning(2, R"(Warning - Parsing ZMapInfo "{}": Unknown token "{}")", entry->name(), tz.current().text);
		}

		tz.adv();
	}

	Log::info(2, "Parsed ZMapInfo entry {} successfully", entry->name());

	return true;
}

// -----------------------------------------------------------------------------
// Parses a ZMAPINFO map definition of [type] beginning at the current token in
// tokenizer [tz]
// -----------------------------------------------------------------------------
bool MapInfo::parseZMap(Tokenizer& tz, std::string_view type)
{
	// TODO: Handle adddefaultmap
	auto map = default_map_;

	// Normal map, get lump/name/etc
	tz.adv();
	if (type == "map")
	{
		// Entry name should be just after map keyword
		map.entry_name = tz.current().text;

		// Parse map name
		tz.adv();
		if (tz.check("lookup"))
		{
			map.lookup_name = true;
			map.name        = tz.next().text;
		}
		else
		{
			map.lookup_name = false;
			map.name        = tz.current().text;
		}

		tz.adv();
	}

	if (!tz.advIf("{"))
//.........这里部分代码省略.........

QByteArray qsgShaderRewriter_insertZAttributes(const char *input, QSurfaceFormat::OpenGLContextProfile profile)
{
    Tokenizer tok;
    tok.initialize(input);

    Tokenizer::Token lt = tok.next();
    Tokenizer::Token t = tok.next();

    // First find "void main() { ... "
    const char* voidPos = input;
    while (t != Tokenizer::Token_EOF) {
        if (lt == Tokenizer::Token_Void && t == Tokenizer::Token_Identifier) {
            if (qstrncmp("main", tok.identifier, 4) == 0)
                break;
        }
        voidPos = tok.pos - 4;
        lt = t;
        t = tok.next();
    }

    QByteArray result;
    result.reserve(1024);
    result += QByteArray::fromRawData(input, voidPos - input);
    switch (profile) {
    case QSurfaceFormat::NoProfile:
    case QSurfaceFormat::CompatibilityProfile:
        result += QByteArrayLiteral("attribute highp float _qt_order;\n");
        result += QByteArrayLiteral("uniform highp float _qt_zRange;\n");
        break;

    case QSurfaceFormat::CoreProfile:
        result += QByteArrayLiteral("in float _qt_order;\n");
        result += QByteArrayLiteral("uniform float _qt_zRange;\n");
        break;
    }

    // Find first brace '{'
    while (t != Tokenizer::Token_EOF && t != Tokenizer::Token_OpenBrace) t = tok.next();
    int braceDepth = 1;
    t = tok.next();

    // Find matching brace and insert our code there...
    while (t != Tokenizer::Token_EOF) {
        switch (t) {
        case Tokenizer::Token_CloseBrace:
            braceDepth--;
            if (braceDepth == 0) {
                result += QByteArray::fromRawData(voidPos, tok.pos - 1 - voidPos);
                result += QByteArrayLiteral("    gl_Position.z = (gl_Position.z * _qt_zRange + _qt_order) * gl_Position.w;\n");
                result += QByteArray(tok.pos - 1);
                return result;
            }
            break;
        case Tokenizer::Token_OpenBrace:
            ++braceDepth;
            break;
        default:
            break;
        }
        t = tok.next();
    }
    return QByteArray();
}

//.........这里部分代码省略.........
	string name, type;
	while (!tz.atEnd() && tz.current() != '}')
	{
		// Check for preprocessor stuff
		if (parser_ && tz.current()[0] == '#')
		{
			if (!parsePreprocessor(tz))
				return false;

			tz.advToNextLine();
			continue;
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(tz.current().text[0]))
		{
			logError(tz, fmt::format("Unexpected special character '{}'", tz.current().text));
			return false;
		}

		// So we have either a node or property name
		name = tz.current().text;
		type.clear();
		if (name.empty())
		{
			logError(tz, "Unexpected empty string");
			return false;
		}

		// Check for type+name pair
		if (tz.peek() != '=' && tz.peek() != '{' && tz.peek() != ';' && tz.peek() != ':')
		{
			type = name;
			name = tz.next().text;

			if (name.empty())
			{
				logError(tz, "Unexpected empty string");
				return false;
			}
		}

		// Log::debug(wxString::Format("%s \"%s\", op %s", CHR(type), CHR(name), CHR(tz.current().text)));

		// Assignment
		if (tz.advIfNext('=', 2))
		{
			if (!parseAssignment(tz, addChildPTN(name, type)))
				return false;
		}

		// Child node
		else if (tz.advIfNext('{', 2))
		{
			// Parse child node
			if (!addChildPTN(name, type)->parse(tz))
				return false;
		}

		// Child node (with no values/children)
		else if (tz.advIfNext(';', 2))
		{
			// Add child node
			addChildPTN(name, type);
			continue;
		}