C++ Tokenizer::getToken方法代码示例

void ArchiveOperations::removeUnusedFlats(Archive* archive)
{
	// Check archive was given
	if (!archive)
		return;

	// --- Build list of used flats ---
	TexUsedMap used_textures;
	int        total_maps = 0;

	// Get all SECTORS entries
	Archive::SearchOptions opt;
	opt.match_type = EntryType::fromId("map_sectors");
	auto sectors   = archive->findAll(opt);
	total_maps += sectors.size();

	// Go through and add used flats to list
	DoomMapFormat::Sector sec;
	wxString              tex_floor, tex_ceil;
	for (auto& sector : sectors)
	{
		int nsec = sector->size() / 26;
		sector->seek(0, SEEK_SET);
		for (int s = 0; s < nsec; s++)
		{
			// Read sector data
			sector->read(&sec, 26);

			// Get textures
			tex_floor = wxString::FromAscii(sec.f_tex, 8);
			tex_ceil  = wxString::FromAscii(sec.c_tex, 8);

			// Add to used textures list
			used_textures[tex_floor].used = true;
			used_textures[tex_ceil].used  = true;
		}
	}

	// Get all TEXTMAP entries
	opt.match_name = "TEXTMAP";
	opt.match_type = EntryType::fromId("udmf_textmap");
	auto udmfmaps  = archive->findAll(opt);
	total_maps += udmfmaps.size();

	// Go through and add used flats to list
	Tokenizer tz;
	tz.setSpecialCharacters("{};=");
	for (auto& udmfmap : udmfmaps)
	{
		// Open in tokenizer
		tz.openMem(udmfmap->data(), "UDMF TEXTMAP");

		// Go through text tokens
		wxString token = tz.getToken();
		while (!token.IsEmpty())
		{
			// Check for sector definition
			if (token == "sector")
			{
				tz.getToken(); // Skip {

				token = tz.getToken();
				while (token != "}")
				{
					// Check for texture property
					if (token == "texturefloor" || token == "textureceiling")
					{
						tz.getToken(); // Skip =
						used_textures[tz.getToken()].used = true;
					}

					token = tz.getToken();
				}
			}

			// Next token
			token = tz.getToken();
		}
	}

	// Check if any maps were found
	if (total_maps == 0)
		return;

	// Find all flats
	opt.match_name      = "";
	opt.match_namespace = "flats";
	opt.match_type      = nullptr;
	auto flats          = archive->findAll(opt);

	// Create list of all unused flats
	wxArrayString unused_tex;
	bool          anim = false;
	for (auto& flat : flats)
	{
		// Skip markers
		if (flat->size() == 0)
			continue;

		// Check for animation start
//.........这里部分代码省略.........

 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *
 */
#include "tokenizer.hpp"
#include <sstream>
#include "catch.hpp"

TEST_CASE( "Parsing Tokens", "[Tokenizer::parse]" ) {
	Tokenizer tok;
	std::stringstream test;
	test <<
		"test_project {" << std::endl <<
		"test=1," << std::endl <<
		"	 meep = 39" << std::endl <<
		"}" << std::endl;

	tok = Tokenizer(test);
	CHECK(tok.next() == "test_project");
	Token token = tok.getToken();
	CHECK_FALSE(token.getType() == ERROR);
	CHECK(token.getType() == PROJECT);
	CHECK(token.getValue() == "test_project");
}

void ArchiveOperations::removeUnusedTextures(Archive* archive)
{
	// Check archive was given
	if (!archive)
		return;

	// --- Build list of used textures ---
	TexUsedMap used_textures;
	int        total_maps = 0;

	// Get all SIDEDEFS entries
	Archive::SearchOptions opt;
	opt.match_type = EntryType::fromId("map_sidedefs");
	auto sidedefs  = archive->findAll(opt);
	total_maps += sidedefs.size();

	// Go through and add used textures to list
	DoomMapFormat::SideDef sdef;
	wxString               tex_lower, tex_middle, tex_upper;
	for (auto& sidedef : sidedefs)
	{
		int nsides = sidedef->size() / 30;
		sidedef->seek(0, SEEK_SET);
		for (int s = 0; s < nsides; s++)
		{
			// Read side data
			sidedef->read(&sdef, 30);

			// Get textures
			tex_lower  = wxString::FromAscii(sdef.tex_lower, 8);
			tex_middle = wxString::FromAscii(sdef.tex_middle, 8);
			tex_upper  = wxString::FromAscii(sdef.tex_upper, 8);

			// Add to used textures list
			used_textures[tex_lower].used  = true;
			used_textures[tex_middle].used = true;
			used_textures[tex_upper].used  = true;
		}
	}

	// Get all TEXTMAP entries
	opt.match_name = "TEXTMAP";
	opt.match_type = EntryType::fromId("udmf_textmap");
	auto udmfmaps  = archive->findAll(opt);
	total_maps += udmfmaps.size();

	// Go through and add used textures to list
	Tokenizer tz;
	tz.setSpecialCharacters("{};=");
	for (auto& udmfmap : udmfmaps)
	{
		// Open in tokenizer
		tz.openMem(udmfmap->data(), "UDMF TEXTMAP");

		// Go through text tokens
		wxString token = tz.getToken();
		while (!token.IsEmpty())
		{
			// Check for sidedef definition
			if (token == "sidedef")
			{
				tz.getToken(); // Skip {

				token = tz.getToken();
				while (token != "}")
				{
					// Check for texture property
					if (token == "texturetop" || token == "texturemiddle" || token == "texturebottom")
					{
						tz.getToken(); // Skip =
						used_textures[tz.getToken()].used = true;
					}

					token = tz.getToken();
				}
			}

			// Next token
			token = tz.getToken();
		}
	}

	// Check if any maps were found
	if (total_maps == 0)
		return;

	// Find all TEXTUREx entries
	opt.match_name  = "";
	opt.match_type  = EntryType::fromId("texturex");
	auto tx_entries = archive->findAll(opt);

	// Go through texture lists
	PatchTable    ptable; // Dummy patch table, patch info not needed here
	wxArrayString unused_tex;
	for (auto& tx_entrie : tx_entries)
	{
		TextureXList txlist;
		txlist.readTEXTUREXData(tx_entrie, ptable);

		// Go through textures
//.........这里部分代码省略.........

/* MapPreviewCanvas::openMap
 * Opens a map from a mapdesc_t
 *******************************************************************/
bool MapPreviewCanvas::openMap(Archive::mapdesc_t map)
{
	// All errors = invalid map
	Global::error = "Invalid map";

	// Check if this map is a pk3 map
	bool map_archive = false;
	if (map.archive)
	{
		map_archive = true;

		// Attempt to open entry as wad archive
		temp_archive = new WadArchive();
		if (!temp_archive->open(map.head))
		{
			delete temp_archive;
			return false;
		}

		// Detect maps
		vector<Archive::mapdesc_t> maps = temp_archive->detectMaps();

		// Set map if there are any in the archive
		if (maps.size() > 0)
			map = maps[0];
		else
			return false;
	}

	// Parse UDMF map
	if (map.format == MAP_UDMF)
	{
		ArchiveEntry* udmfdata = NULL;
		for (ArchiveEntry* mapentry = map.head; mapentry != map.end; mapentry = mapentry->nextEntry())
		{
			// Check entry type
			if (mapentry->getType() == EntryType::getType("udmf_textmap"))
			{
				udmfdata = mapentry;
				break;
			}
		}
		if (udmfdata == NULL)
			return false;

		// Start parsing
		Tokenizer tz;
		tz.openMem(udmfdata->getData(), udmfdata->getSize(), map.head->getName());

		// Get first token
		string token = tz.getToken();
		size_t vertcounter = 0, linecounter = 0, thingcounter = 0;
		while (!token.IsEmpty())
		{
			if (!token.CmpNoCase("namespace"))
			{
				//  skip till we reach the ';'
				do { token = tz.getToken(); }
				while (token.Cmp(";"));
			}
			else if (!token.CmpNoCase("vertex"))
			{
				// Get X and Y properties
				bool gotx = false;
				bool goty = false;
				double x = 0.;
				double y = 0.;
				do
				{
					token = tz.getToken();
					if (!token.CmpNoCase("x") || !token.CmpNoCase("y"))
					{
						bool isx = !token.CmpNoCase("x");
						token = tz.getToken();
						if (token.Cmp("="))
						{
							wxLogMessage("Bad syntax for vertex %i in UDMF map data", vertcounter);
							return false;
						}
						if (isx) x = tz.getDouble(), gotx = true;
						else y = tz.getDouble(), goty = true;
						// skip to end of declaration after each key
						do { token = tz.getToken(); }
						while (token.Cmp(";"));
					}
				}
				while (token.Cmp("}"));
				if (gotx && goty)
					addVertex(x, y);
				else
				{
					wxLogMessage("Wrong vertex %i in UDMF map data", vertcounter);
					return false;
				}
				vertcounter++;
			}
			else if (!token.CmpNoCase("linedef"))
//.........这里部分代码省略.........

/* ParseTreeNode::parse
 * Parses formatted text data. Current valid formatting is:
 * (type) child = value;
 * (type) child = value1, value2, ...;
 * (type) child = { value1, value2, ... }
 * (type) child { grandchild = value; etc... }
 * (type) child : inherited { ... }
 * All values are read as strings, but can be retrieved as string,
 * int, bool or float.
 *******************************************************************/
bool ParseTreeNode::parse(Tokenizer& tz)
{
	// Get first token
	string token = tz.getToken();

	// Keep parsing until final } is reached (or end of file)
	while (!(S_CMP(token, "}")) && !token.IsEmpty())
	{
		// Check for preprocessor stuff
		if (parser)
		{
			// #define
			if (S_CMPNOCASE(token, "#define"))
			{
				parser->define(tz.getToken());
				token = tz.getToken();
				continue;
			}

			// #if(n)def
			if (S_CMPNOCASE(token, "#ifdef") || S_CMPNOCASE(token, "#ifndef"))
			{
				bool test = true;
				if (S_CMPNOCASE(token, "#ifndef"))
					test = false;
				string define = tz.getToken();
				if (parser->defined(define) == test)
				{
					// Continue
					token = tz.getToken();
					continue;
				}
				else
				{
					// Skip section
					int skip = 0;
					while (true)
					{
						token = tz.getToken();
						if (S_CMPNOCASE(token, "#endif"))
							skip--;
						else if (S_CMPNOCASE(token, "#ifdef"))
							skip++;
						else if (S_CMPNOCASE(token, "#ifndef"))
							skip++;

						if (skip < 0)
							break;
						if (token.IsEmpty())
						{
							wxLogMessage("Error: found end of file within #if(n)def block");
							break;
						}
					}

					continue;
				}
			}

			// #include (ignore)
			if (S_CMPNOCASE(token, "#include"))
			{
				tz.skipToken();	// Skip include path
				token = tz.getToken();
				continue;
			}

			// #endif (ignore)
			if (S_CMPNOCASE(token, "#endif"))
			{
				token = tz.getToken();
				continue;
			}
		}

		// If it's a special character (ie not a valid name), parsing fails
		if (tz.isSpecialCharacter(token.at(0)))
		{
			wxLogMessage("Parsing error: Unexpected special character '%s' in %s (line %d)", token, tz.getName(), tz.lineNo());
			return false;
		}

		// So we have either a node or property name
		string name = token;
		if (name.IsEmpty())
		{
			wxLogMessage("Parsing error: Unexpected empty string in %s (line %d)", tz.getName(), tz.lineNo());
			return false;
		}

//.........这里部分代码省略.........

// -----------------------------------------------------------------------------
// Initialises the start page
// -----------------------------------------------------------------------------
void SStartPage::init()
{
	// wxWebView
#ifdef USE_WEBVIEW_STARTPAGE
	html_startpage_ = wxWebView::New(
		this, -1, wxEmptyString, wxDefaultPosition, wxDefaultSize, wxWebViewBackendDefault, wxBORDER_NONE);
	html_startpage_->SetZoomType(App::platform() == App::MacOS ? wxWEBVIEW_ZOOM_TYPE_TEXT : wxWEBVIEW_ZOOM_TYPE_LAYOUT);

	// wxHtmlWindow
#else
	html_startpage_ = new wxHtmlWindow(this, -1, wxDefaultPosition, wxDefaultSize, wxHW_SCROLLBAR_NEVER, "startpage");
#endif

	// Add to sizer
	GetSizer()->Add(html_startpage_, 1, wxEXPAND);

	// Bind events
#ifdef USE_WEBVIEW_STARTPAGE
	html_startpage_->Bind(wxEVT_WEBVIEW_NAVIGATING, &SStartPage::onHTMLLinkClicked, this);

	html_startpage_->Bind(
		wxEVT_WEBVIEW_ERROR, [&](wxWebViewEvent& e) { Log::error(S_FMT("wxWebView Error: %s", CHR(e.GetString()))); });

	if (App::platform() == App::Platform::Windows)
	{
		html_startpage_->Bind(wxEVT_WEBVIEW_LOADED, [&](wxWebViewEvent& e) { html_startpage_->Reload(); });
	}

	Bind(wxEVT_THREAD_WEBGET_COMPLETED, [&](wxThreadEvent& e) {
		latest_news_ = e.GetString();
		latest_news_.Trim();

		if (latest_news_ == "connect_failed" || latest_news_.empty())
			latest_news_ = "<center>Unable to load latest SLADE news</center>";

		load(false);
	});

#else
	html_startpage_->Bind(wxEVT_COMMAND_HTML_LINK_CLICKED, &SStartPage::onHTMLLinkClicked, this);
#endif

	// Get data used to build the page
	auto res_archive = App::archiveManager().programResourceArchive();
	if (res_archive)
	{
		entry_base_html_ =
			res_archive->entryAtPath(App::useWebView() ? "html/startpage.htm" : "html/startpage_basic.htm");

		entry_css_ = res_archive->entryAtPath(web_dark_theme ? "html/theme-dark.css" : "html/theme-light.css");

		entry_export_.push_back(res_archive->entryAtPath("html/base.css"));
		entry_export_.push_back(res_archive->entryAtPath("fonts/FiraSans-Regular.woff"));
		entry_export_.push_back(res_archive->entryAtPath("fonts/FiraSans-Italic.woff"));
		entry_export_.push_back(res_archive->entryAtPath("fonts/FiraSans-Medium.woff"));
		entry_export_.push_back(res_archive->entryAtPath("fonts/FiraSans-Bold.woff"));
		entry_export_.push_back(res_archive->entryAtPath("fonts/FiraSans-Heavy.woff"));
		entry_export_.push_back(res_archive->entryAtPath("logo_icon.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/entry_list/Rounded/archive.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/entry_list/Rounded/wad.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/entry_list/Rounded/zip.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/entry_list/Rounded/folder.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/general/open.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/general/newarchive.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/general/newzip.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/general/mapeditor.png"));
		entry_export_.push_back(res_archive->entryAtPath("icons/general/wiki.png"));

		// Load tips
		auto entry_tips = res_archive->entryAtPath("tips.txt");
		if (entry_tips)
		{
			Tokenizer tz;
			tz.openMem((const char*)entry_tips->getData(), entry_tips->getSize(), entry_tips->getName());
			while (!tz.atEnd() && tz.peekToken() != "")
				tips_.push_back(tz.getToken());
		}
	}
}

/* JumpToCalculator::Entry
 * JumpToCalculator thread entry function
 *******************************************************************/
wxThread::ExitCode JumpToCalculator::Entry()
{
	string jump_points;

	Tokenizer tz;
	tz.setSpecialCharacters(";,:|={}/()");
	tz.openString(text);

	string token = tz.getToken();
	while (!tz.atEnd())
	{
		if (token == "{")
		{
			// Skip block
			while (!tz.atEnd() && token != "}")
				token = tz.getToken();
		}

		for (unsigned a = 0; a < block_names.size(); a++)
		{
			// Get jump block keyword
			string block = block_names[a];
			long skip = 0;
			if (block.Contains(":"))
			{
				wxArrayString sp = wxSplit(block, ':');
				sp.back().ToLong(&skip);
				block = sp[0];
			}

			if (S_CMPNOCASE(token, block))
			{
				string name = tz.getToken();
				for (int s = 0; s < skip; s++)
					name = tz.getToken();

				for (unsigned i = 0; i < ignore.size(); ++i)
					if (S_CMPNOCASE(name, ignore[i]))
						name = tz.getToken();

				// Numbered block, add block name
				if (name.IsNumber())
					name = S_FMT("%s %s", block, name);
				// Unnamed block, use block name
				if (name == "{" || name == ";")
					name = block;

				// Add jump point
				jump_points += S_FMT("%d,%s,", tz.lineNo() - 1, CHR(name));
			}
		}

		token = tz.getToken();
	}

	// Remove ending comma
	if (!jump_points.empty())
		jump_points.RemoveLast(1);

	// Send event
	wxThreadEvent* event = new wxThreadEvent(wxEVT_COMMAND_JTCALCULATOR_COMPLETED);
	event->SetString(jump_points);
	wxQueueEvent(handler, event);

	return nullptr;
}

/* CTexture::parse
 * Parses a TEXTURES format texture definition
 *******************************************************************/
bool CTexture::parse(Tokenizer& tz, string type)
{
	// Check if optional
	if (S_CMPNOCASE(tz.peekToken(), "optional"))
	{
		tz.getToken();	// Skip it
		optional = true;
	}

	// Read basic info
	this->type = type;
	this->extended = true;
	this->defined = false;
	name = tz.getToken().Upper();
	tz.getToken();	// Skip ,
	width = tz.getInteger();
	tz.getToken();	// Skip ,
	height = tz.getInteger();

	// Check for extended info
	if (tz.peekToken() == "{")
	{
		tz.getToken();	// Skip {

		// Read properties
		string property = tz.getToken();
		while (property != "}")
		{
			// Check if end of text is reached (error)
			if (property.IsEmpty())
			{
				wxLogMessage("Error parsing texture %s: End of text found, missing } perhaps?", name);
				return false;
			}

			// XScale
			if (S_CMPNOCASE(property, "XScale"))
				scale_x = tz.getFloat();

			// YScale
			if (S_CMPNOCASE(property, "YScale"))
				scale_y = tz.getFloat();

			// Offset
			if (S_CMPNOCASE(property, "Offset"))
			{
				offset_x = tz.getInteger();
				tz.getToken();	// Skip ,
				offset_y = tz.getInteger();
			}

			// WorldPanning
			if (S_CMPNOCASE(property, "WorldPanning"))
				world_panning = true;

			// NoDecals
			if (S_CMPNOCASE(property, "NoDecals"))
				no_decals = true;

			// NullTexture
			if (S_CMPNOCASE(property, "NullTexture"))
				null_texture = true;

			// Patch
			if (S_CMPNOCASE(property, "Patch"))
			{
				CTPatchEx* patch = new CTPatchEx();
				patch->parse(tz);
				patches.push_back(patch);
			}

			// Graphic
			if (S_CMPNOCASE(property, "Graphic"))
			{
				CTPatchEx* patch = new CTPatchEx();
				patch->parse(tz, PTYPE_GRAPHIC);
				patches.push_back(patch);
			}

			// Read next property
			property = tz.getToken();
		}
	}

	return true;
}

/* TextureXList::readTEXTURESData
 * Reads in a ZDoom-format TEXTURES entry. Returns true on success,
 * false otherwise
 *******************************************************************/
bool TextureXList::readTEXTURESData(ArchiveEntry* entry)
{
	// Check for empty entry
	if (!entry)
	{
		Global::error = "Attempt to read texture data from NULL entry";
		return false;
	}
	if (entry->getSize() == 0)
	{
		txformat = TXF_TEXTURES;
		return true;
	}

	// Get text to parse
	Tokenizer tz;
	tz.openMem(&(entry->getMCData()), entry->getName());

	// Parsing gogo
	string token = tz.getToken();
	while (!token.IsEmpty())
	{
		// Texture definition
		if (S_CMPNOCASE(token, "Texture"))
		{
			CTexture* tex = new CTexture();
			if (tex->parse(tz, "Texture"))
				addTexture(tex);
		}

		// Sprite definition
		if (S_CMPNOCASE(token, "Sprite"))
		{
			CTexture* tex = new CTexture();
			if (tex->parse(tz, "Sprite"))
				addTexture(tex);
		}

		// Graphic definition
		if (S_CMPNOCASE(token, "Graphic"))
		{
			CTexture* tex = new CTexture();
			if (tex->parse(tz, "Graphic"))
				addTexture(tex);
		}

		// WallTexture definition
		if (S_CMPNOCASE(token, "WallTexture"))
		{
			CTexture* tex = new CTexture();
			if (tex->parse(tz, "WallTexture"))
				addTexture(tex);
		}

		// Flat definition
		if (S_CMPNOCASE(token, "Flat"))
		{
			CTexture* tex = new CTexture();
			if (tex->parse(tz, "Flat"))
				addTexture(tex);
		}

		token = tz.getToken();
	}

	txformat = TXF_TEXTURES;

	return true;
}

/* Console::execute
 * Attempts to execute the command line given
 *******************************************************************/
void Console::execute(string command)
{
	wxLogMessage("> %s", command);

	// Don't bother doing anything else with an empty command
	if (command.size() == 0)
		return;

	// Add the command to the log
	cmd_log.insert(cmd_log.begin(), command);

	// Announce that a command has been executed
	MemChunk mc;
	announce("console_execute", mc);

	// Tokenize the command string
	Tokenizer tz;
	tz.openString(command);

	// Get the command name
	string cmd_name = tz.getToken();

	// Get all args
	string arg = tz.getToken();
	vector<string> args;
	while (arg != "")
	{
		args.push_back(arg);
		arg = tz.getToken();
	}

	// Check that it is a valid command
	for (size_t a = 0; a < commands.size(); a++)
	{
		// Found it, execute and return
		if (commands[a].getName() == cmd_name)
		{
			commands[a].execute(args);
			return;
		}
	}

	// Check if it is a cvar
	CVar* cvar = get_cvar(cmd_name);
	if (cvar)
	{
		// Arg(s) given, set cvar value
		if (args.size() > 0)
		{
			if (cvar->type == CVAR_BOOLEAN)
			{
				if (args[0] == "0" || args[0] == "false")
					*((CBoolCVar*)cvar) = false;
				else
					*((CBoolCVar*)cvar) = true;
			}
			else if (cvar->type == CVAR_INTEGER)
				*((CIntCVar*)cvar) = atoi(CHR(args[0]));
			else if (cvar->type == CVAR_FLOAT)
				*((CFloatCVar*)cvar) = (float)atof(CHR(args[0]));
			else if (cvar->type == CVAR_STRING)
				*((CStringCVar*)cvar) = args[0];
		}

		// Print cvar value
		string value = "";
		if (cvar->type == CVAR_BOOLEAN)
		{
			if (cvar->GetValue().Bool)
				value = "true";
			else
				value = "false";
		}
		else if (cvar->type == CVAR_INTEGER)
			value = S_FMT("%d", cvar->GetValue().Int);
		else if (cvar->type == CVAR_FLOAT)
			value = S_FMT("%1.4f", cvar->GetValue().Float);
		else
			value = ((CStringCVar*)cvar)->value;

		logMessage(S_FMT("\"%s\" = \"%s\"", cmd_name, value));

		if (cmd_name == "log_verbosity")
			Global::log_verbosity = cvar->GetValue().Int;

		return;
	}

	// Toggle global debug mode
	if (cmd_name == "debug")
	{
		Global::debug = !Global::debug;
		if (Global::debug)
			logMessage("Debugging stuff enabled");
		else
			logMessage("Debugging stuff disabled");

//.........这里部分代码省略.........

void Stmt::printStmt()
{
  curToken=t.getToken();
  if(curToken == 32){
    cout<<t.getIdName();
    t.skipToken();
    curToken=t.getToken();
#ifdef ENABLE_DEBUG_PRINT
    cout<<" curToken-printStmt- Var "<<curToken<<endl;
#endif
    assign.printAssign();
  }
  else if(curToken==5) {
    cout<<" if ";
    t.skipToken();
    cond.printCondition();
    curToken=t.getToken();
    while(curToken != 3) {
      this->getIfInstanceStmt()->printIF();
      t.skipToken();
      curToken=t.getToken();
#ifdef ENABLE_DEBUG_PRINT
      cout<<" curToken-printStmt- if "<<curToken<<endl;
#endif
      if(curToken == 6){ cout<<" then "<<endl; t.skipToken(); }
      else if(curToken == 3) { cout<<" end "; }
      else if(curToken == 12) { cout<<" ; "<<endl; }
    }
  }
  else if(curToken==8) {
    cout<<" while ";
    t.skipToken();
    curToken=t.getToken();
    while(curToken != 3){
      this->getLoopInstanceStmt()->printLoop();
      t.skipToken();
      curToken=t.getToken();
#ifdef ENABLE_DEBUG_PRINT
      cout<<" curToken-printStmt- while "<<curToken<<endl;
#endif
      if(curToken == 9){ cout<<" loop "<<endl; t.skipToken(); }
      else if(curToken == 3) { cout<<" end "; }
      else if(curToken == 12) { cout<<" ; "<<endl; }
    }
  }
  else if(curToken==10) {
    cout<<" read ";
    t.skipToken();
    curToken=t.getToken();
    while(curToken != 12){
      in.printIN();
      t.skipToken();
      curToken=t.getToken();
#ifdef ENABLE_DEBUG_PRINT
      cout<<" curToken-printStmt- read "<<curToken<<endl;
#endif
      if(curToken==12) { cout<<"; "<<endl; }
      else if(curToken==13) { cout<<", "; }
      else if(curToken==14) { cout<<" = "; }
    }
  }
  else if(curToken==11) {
    cout<<" write ";
    t.skipToken();
    curToken=t.getToken();
#ifdef ENABLE_DEBUG_PRINT
    cout<<" curToken-printStmt- write "<<curToken<<endl;
#endif
    out.printOUT();
  }
}

/* MapPreviewCanvas::openMap
 * Opens a map from a mapdesc_t
 *******************************************************************/
bool MapPreviewCanvas::openMap(Archive::mapdesc_t map)
{
	// All errors = invalid map
	Global::error = "Invalid map";

	// Check if this map is a pk3 map
	bool map_archive = false;
	if (map.archive)
	{
		map_archive = true;

		// Attempt to open entry as wad archive
		temp_archive = new WadArchive();
		if (!temp_archive->open(map.head))
		{
			delete temp_archive;
			return false;
		}

		// Detect maps
		vector<Archive::mapdesc_t> maps = temp_archive->detectMaps();

		// Set map if there are any in the archive
		if (maps.size() > 0)
			map = maps[0];
		else
			return false;
	}

	// Parse UDMF map
	if (map.format == MAP_UDMF)
	{
		ArchiveEntry* udmfdata = NULL;
		for (ArchiveEntry* mapentry = map.head; mapentry != map.end; mapentry = mapentry->nextEntry())
		{
			// Check entry type
			if (mapentry->getType() == EntryType::getType("udmf_textmap"))
			{
				udmfdata = mapentry;
				break;
			}
		}
		if (udmfdata == NULL)
			return false;

		// Start parsing
		Tokenizer tz;
		tz.openMem(udmfdata->getData(), udmfdata->getSize(), map.head->getName());

		// Get first token
		string token = tz.getToken();
		size_t vertcounter = 0, linecounter = 0;
		while (!token.IsEmpty())
		{
			if (!token.CmpNoCase("namespace"))
			{
				//  skip till we reach the ';'
				do { token = tz.getToken(); }
				while (token.Cmp(";"));
			}
			else if (!token.CmpNoCase("vertex"))
			{
				// Get X and Y properties
				bool gotx = false;
				bool goty = false;
				double x = 0.;
				double y = 0.;
				do
				{
					token = tz.getToken();
					if (!token.CmpNoCase("x") || !token.CmpNoCase("y"))
					{
						bool isx = !token.CmpNoCase("x");
						token = tz.getToken();
						if (token.Cmp("="))
						{
							wxLogMessage("Bad syntax for vertex %i in UDMF map data", vertcounter);
							return false;
						}
						if (isx) x = tz.getDouble(), gotx = true;
						else y = tz.getDouble(), goty = true;
						// skip to end of declaration after each key
						do { token = tz.getToken(); }
						while (token.Cmp(";"));
					}
				}
				while (token.Cmp("}"));
				if (gotx && goty)
					addVertex(x, y);
				else
				{
					wxLogMessage("Wrong vertex %i in UDMF map data", vertcounter);
					return false;
				}
				vertcounter++;
			}
			else if (!token.CmpNoCase("linedef"))
//.........这里部分代码省略.........

void InvertedIndex::indexing(Tokenizer& t, int index){
	unordered_set<string> docs_words;

	/* A file must be indexed entirely, so it is possible to calculate total number of terms in a same file.
		To do so, it is calculate the necessaary space to store this one file and the actual size of index.
		If it is going to be bigger than memory size, dump index first.
	*/
	if ((this->memory_usage) && (((t.size()*INDEX_LINE_SIZE)+this->memory_usage) >= MEMORY_LIMITE)){
		this->memory_dump();
	}

	if (t.size() > 0){
		this->total_docs++;
	}

	// Iterating through tokens
	int word_id = 0;
	while (t.size() > 0){
		string token = t.getToken();

		if (token.size()) {
			this->total_token++;

			docs_words.insert(token);

			// Testing if token is not already in the vocabulary
			auto search = this->vocabulary.find(token);
			if (search == this->vocabulary.end()){

				// Adding to vocabulary hash
				this->vocabulary[token] = this->word_index;

				// Saving group of vocabulary info
				Vocabulary item;
				item.word = token;
				item.id = this->word_index;
				item.file_pos = 0;		// Unknown right now
				item.total_docs = 0;	// Unknown right now
				item.idf = -1;			// Unknown right now

				this->vocabulary_order.push_back(item);

				// Updating word index in the vocabulary
				this->word_index++;
			}

			// Testing if token is not already in the index
			auto search2 = this->inverted_index.find(token);

			// Testing if token had already been seen in document index
			if (search2 != this->inverted_index.end() &&					// Testing if token is in index
				this->inverted_index[token].size() > 0 &&
				this->inverted_index[token].back().file_index == index) {
					this->inverted_index[token].back().position.push_back(word_id);
			}
			else {
				FileList list;
				list.file_index = index;
				list.position.push_back(word_id);

				this->inverted_index[token].push_back(list);
			}

			this->memory_usage+=INDEX_LINE_SIZE;

			word_id++;

			if (this->memory_usage >= MEMORY_LIMITE){
				this->memory_dump();
			}
		}
	}

	// Updating n_i
	for (string word : docs_words){
		this->vocabulary_order[this->vocabulary[word]].total_docs++;
	}
}

/* MainWindow::createStartPage
 * Builds the HTML start page and loads it into the html viewer
 * (start page tab)
 *******************************************************************/
void MainWindow::createStartPage(bool newtip)
{
	// Get relevant resource entries
	Archive* res_archive = theArchiveManager->programResourceArchive();
	if (!res_archive)
		return;

	// Get entries to export
	vector<ArchiveEntry*> export_entries;
	ArchiveEntry* entry_html = res_archive->entryAtPath("html/startpage.htm");
	ArchiveEntry* entry_tips = res_archive->entryAtPath("tips.txt");
	export_entries.push_back(res_archive->entryAtPath("logo.png"));
	export_entries.push_back(res_archive->entryAtPath("html/box-title-back.png"));

	// Can't do anything without html entry
	if (!entry_html)
	{
		LOG_MESSAGE(1, "No start page resource found");
		html_startpage->SetPage("<html><head><title>SLADE</title></head><body><center><h1>Something is wrong with slade.pk3 :(</h1><center></body></html>", wxEmptyString);
		return;
	}

	// Get html as string
	string html = wxString::FromAscii((const char*)(entry_html->getData()), entry_html->getSize());

	// Generate tip of the day string
	string tip = "It seems tips.txt is missing from your slade.pk3";
	if (entry_tips)
	{
		Tokenizer tz;
		tz.openMem((const char*)entry_tips->getData(), entry_tips->getSize(), entry_tips->getName());
		srand(wxGetLocalTime());
		int numtips = tz.getInteger();
		if (numtips < 2) // Needs at least two choices or it's kinda pointless.
			tip = "Did you know? Something is wrong with the tips.txt file in your slade.pk3.";
		else
		{
			int tipindex = lasttipindex;
			if (newtip || lasttipindex == 0)
			{
				// Don't show same tip twice in a row
				do { tipindex = 1 + (rand() % numtips); } while (tipindex == lasttipindex);
			}
			
			lasttipindex = tipindex;
			for (int a = 0; a < tipindex; a++)
				tip = tz.getToken();
		}
	}

	// Generate recent files string
	string recent;
	recent += "<table class=\"box\">";
	if (theArchiveManager->numRecentFiles() > 0)
	{
		for (unsigned a = 0; a < 12; a++)
		{
			if (a >= theArchiveManager->numRecentFiles())
				break;	// No more recent files

			recent += "<tr><td valign=\"middle\" class=\"box\">";

			// Determine icon
			string fn = theArchiveManager->recentFile(a);
			string icon = "archive";
			if (fn.EndsWith(".wad"))
				icon = "wad";
			else if (fn.EndsWith(".zip") || fn.EndsWith(".pk3") || fn.EndsWith(".pke"))
				icon = "zip";
			else if (wxDirExists(fn))
				icon = "folder";

			// Add recent file link
			recent += S_FMT("<img src=\"%s.png\"></td><td valign=\"top\" class=\"box\">", icon);
			recent += S_FMT("<a href=\"recent://%d\">%s</a></td></tr>", a, fn);
		}
	}
	else
		recent += "<tr><td valign=\"top\" class=\"box\">No recently opened files</td></tr>";
	recent += "</table>";

	// Insert tip and recent files into html
	html.Replace("#recent#", recent);
	html.Replace("#totd#", tip);

	// Write html and images to temp folder
	for (unsigned a = 0; a < export_entries.size(); a++)
		export_entries[a]->exportFile(App::path(export_entries[a]->getName(), App::Dir::Temp));
	Icons::exportIconPNG(Icons::ENTRY, "archive", App::path("archive.png", App::Dir::Temp));
	Icons::exportIconPNG(Icons::ENTRY, "wad", App::path("wad.png", App::Dir::Temp));
	Icons::exportIconPNG(Icons::ENTRY, "zip", App::path("zip.png", App::Dir::Temp));
	Icons::exportIconPNG(Icons::ENTRY, "folder", App::path("folder.png", App::Dir::Temp));
	string html_file = App::path("startpage.htm", App::Dir::Temp);
	wxFile outfile(html_file, wxFile::write);
	outfile.Write(html);
	outfile.Close();
//.........这里部分代码省略.........

/* MainApp::readConfigFile
 * Reads and parses the SLADE configuration file
 *******************************************************************/
void MainApp::readConfigFile()
{
	// Open SLADE.cfg
	Tokenizer tz;
	if (!tz.openFile(appPath("slade3.cfg", DIR_USER)))
		return;

	// Go through the file with the tokenizer
	string token = tz.getToken();
	while (token.Cmp(""))
	{
		// If we come across a 'cvars' token, read in the cvars section
		if (!token.Cmp("cvars"))
		{
			token = tz.getToken();	// Skip '{'

			// Keep reading name/value pairs until we hit the ending '}'
			string cvar_name = tz.getToken();
			while (cvar_name.Cmp("}"))
			{
				string cvar_val = tz.getToken();
				read_cvar(cvar_name, cvar_val);
				cvar_name = tz.getToken();
			}
		}

		// Read base resource archive paths
		if (!token.Cmp("base_resource_paths"))
		{
			// Skip {
			token = wxString::FromUTF8(UTF8(tz.getToken()));

			// Read paths until closing brace found
			token = tz.getToken();
			while (token.Cmp("}"))
			{
				theArchiveManager->addBaseResourcePath(token);
				token = wxString::FromUTF8(UTF8(tz.getToken()));
			}
		}

		// Read recent files list
		if (token == "recent_files")
		{
			// Skip {
			token = tz.getToken();

			// Read files until closing brace found
			token = wxString::FromUTF8(UTF8(tz.getToken()));
			while (token != "}")
			{
				theArchiveManager->addRecentFile(token);
				token = wxString::FromUTF8(UTF8(tz.getToken()));
			}
		}

		// Read keybinds
		if (token == "keys")
		{
			token = tz.getToken();	// Skip {
			KeyBind::readBinds(tz);
		}

		// Read nodebuilder paths
		if (token == "nodebuilder_paths")
		{
			token = tz.getToken();	// Skip {

			// Read paths until closing brace found
			token = tz.getToken();
			while (token != "}")
			{
				string path = tz.getToken();
				NodeBuilders::addBuilderPath(token, path);
				token = tz.getToken();
			}
		}

		// Read game exe paths
		if (token == "executable_paths")
		{
			token = tz.getToken();	// Skip {

			// Read paths until closing brace found
			token = tz.getToken();
			while (token != "}")
			{
				if (token.length())
				{
					string path = tz.getToken();
					Executables::setExePath(token, path);
				}
				token = tz.getToken();
			}
		}

		// Read window size/position info
//.........这里部分代码省略.........