C++ Toker::getTok方法代码示例

int main()
{
  Helper::Title("Testing Tokenizer");

  //std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
  //std::string fileSpec = "../Tokenizer/Tokenizer.h";
  std::string fileSpec = "../Tokenizer/Test.txt";//from here

  try
  {
    std::ifstream in(fileSpec);
    if (!in.good())
    {
      std::cout << "\n  can't open " << fileSpec << "\n\n";
      return 1;
    }

    {
      Toker toker;
      toker.returnComments();
      toker.attach(&in);
      std::cout << "\n  current line count = " << toker.currentLineCount();
      do
      {
        std::string tok = toker.getTok();
        if (tok == "\n")
          tok = "newline";
        std::cout << "\n -- " << tok;
      } while (in.good());
      std::cout << "\n  current line count = " << toker.currentLineCount();
    }
 
    putline();
    Helper::title("Testing change of special characters");
    std::string newSpecialChars = "., :, +, +=, \n { }";

    Toker toker;
    toker.returnComments();
    toker.setSpecialTokens(newSpecialChars);
    in.clear();

    in.seekg(std::ios::beg);
    toker.attach(&in);
    std::cout << "\n  new special tokens: " << newSpecialChars;
    do
    {
      std::string tok = toker.getTok();
      if (tok == "\n")
        tok = "newline";
      std::cout << "\n -- " << tok;//check if tok is in type table
    } while (in.good());// if there , get filename and say this file is dependent on this file
    std::cout << "\n  current line count = " << toker.currentLineCount();
  }
  catch (std::logic_error& ex)
  {
    std::cout << "\n  " << ex.what();
  }
  std::cout << "\n\n";//here
  return 0;
}

void CodeAnalyzer::AnalyzeFileDepedency(std::string file) {
	Toker toker;
	std::fstream in(file);
	// Check if the filename is valid
	if (!in.good())
	{
		std::cout << "\n can't open file " << file << "\n\n";
		return;
	}

	toker.attach(&in);
	do
	{
		std::string tok = toker.getTok();
		ProcessTok(file, tok);										// Check for match in Final Type Table and perform corresponding action
	} while (in.good());
}

int dealTokenizer(Toker& toker)
{
	std::cout << "Input for Tokenizer" << std::endl;
	std::string fileSpec = "../Tokenizer/test_tokenizer.txt";
	std::ifstream file("../Tokenizer/test_tokenizer.txt");

	std::string str;
	while (std::getline(file, str))
	{
		std::cout << str << std::endl;
	}

	std::cout << "-----------------------------------------------------------------------------------------\n";

	std::ifstream in(fileSpec);
	if (!in.good())
	{
		std::cout << "\n  can't open " << fileSpec << "\n\n";
		return 1;
	}
	
	toker.attach(&in);
	toker.setComments(true);
	toker.set("<,>,[,],(,),{,},:,=,+,-,*,\n", "<,<,>,>,:,:,+,+,-,-,=,=,+,=,-,=,*,=,/,=,-,>");
	std::cout << "OUPUT \n";
	do
	{
		std::string tok = toker.getTok();
		if (tok == "\n")
		{
			tok = "newline";
		
		}
		std::cout << "\n -- " << tok;
	 
	} while (in.good());
	std::cout << "\n";
	
	return 0;
}

int main()
{
  //std::string fileSpec = "../Tokenizer/Tokenizer.cpp";
  //std::string fileSpec = "../Tokenizer/Tokenizer.h";
  std::string fileSpec = "../Tokenizer/Test.txt";

  std::ifstream in(fileSpec);
  if (!in.good())
  {
    std::cout << "\n  can't open " << fileSpec << "\n\n";
    return 1;
  }
  Toker toker;
  toker.attach(&in);
  while (in.good())
  {
    std::string tok = toker.getTok();
    if (tok == "\n")
      tok = "newline";
    std::cout << "\n -- " << tok;
  }
  std::cout << "\n\n";
  return 0;
}

int main()
{
	/*std::fstream fIn;
	fIn.open("C:\\Users\thzha_000\Desktop\finaltest.txt", std::ios::in);*/

	//Tokenizer testing
	std::string fileSpec = "../Tokenizer/test_R4.txt";

	std::cout << "---------------------------------------" << "\n";
	std::cout << "---------  Testing Tokenizer ----------" << "\n";
	std::cout << "---------------------------------------" << "\n";
	std::cout << "--------Operating authentic code-------" << "\n";
	std::cout << "Requirements 5 is well shown in this process" << "\n\n";

	std::ifstream in(fileSpec);
	if (!in.good())
	{
		std::cout << "\n  can't open " << fileSpec << "\n\n";
		return 1;
	}
	else
		std::cout << "\n Read file successfull!" << "\n";
	Toker toker;
	toker.attach(&in);
	do
	{
		std::string tok = toker.getTok();
		if (tok == "\n")
			tok = "newline";
		std::cout << "\n -- " << tok;
	} while (in.good());
	std::cout << "\n\n";

	toker.reset();


	// SemiExp testing
	Toker toker2;
	std::cout << "---------------------------------------" << "\n";
	std::cout << "------- Testing SemiExpression --------" << "\n";
	std::cout << "---------------------------------------" << "\n";
	std::cout << "--------Operating authentic code-------" << "\n\n";
	std::cout << "Requirements 6-8 is well shown in this process" << "\n\n";

	std::fstream in2(fileSpec);
	if (!in2.good())
	{
		std::cout << "\n  can't open file " << fileSpec << "\n\n";
		return 1;
	}
	else
		std::cout << "\n Read file successfull!" << "\n";
	toker2.attach(&in2);
	SemiExp semi(&toker2);
	while (semi.get())
	{
		std::cout << "\n  -------------- semiExpression --------------";
		semi.show();
	}

	/*
	May have collected tokens, but reached end of stream
	before finding SemiExp terminator.
	*/
	if (semi.length() > 0)
	{
		std::cout << "\n  -------------- semiExpression --------------";
		semi.show();
		std::cout << "\n\n";
	}

	toker2.reset();

	// Test Requirement 4 

	std::string fileSpec_R4 = "../Tokenizer/test_R4.txt";

	std::vector<std::string> alter_setSpecialSingleChar = { "=", "+" };
	std::vector<std::string> alter_setSpecialDoubleChar = { "++" , "--" };

	std::cout << "------------------- Requiement 4 ------------------" << "\n";
	std::cout << "---------------------------------------------------" << "\n";
	std::cout << "------Using a small piece of text for testing------" << "\n\n";
	Toker toker1;
	std::ifstream in1(fileSpec_R4);
	if (!in1.good())
	{
		std::cout << "\n  can't open " << fileSpec << "\n\n";
		return 1;
	}
	else
		std::cout << "\n Read file successfull!" << "\n";
	toker.attach(&in1);
	do
	{
		std::string tok = toker1.getTok();
		if (tok == "\n")
			tok = "newline";
		std::cout << "\n -- " << tok;
	} while (in1.good());
//.........这里部分代码省略.........

int main(int argc, char* argv[])
{
	std::cout << "\n  Testing Tokenizer class\n "
		<< std::string(25, '=') << std::endl;
	std::cout
		<< "\n  Note that comments and quotes are returned as single tokens\n\n";

	// collecting tokens from a string

	Toker t_fromStr("tokens from a string: -> int x; /* a comment */", false);
	std::string tok;
	do
	{
		tok = t_fromStr.getTok();
		std::cout << "  " << tok;
	} while (tok != "");
	std::cout << "\n\n";

	// collecting tokens from files, named on the command line

	if (argc < 2)
	{
		std::cout
			<< "\n  please enter name of file to process on command line\n\n";
		return 1;
	}

	for (int i = 1; i<argc; ++i)
	{
		std::cout << "\n  Processing file " << argv[i];
		std::cout << "\n  " << std::string(16 + strlen(argv[i]), '-') << "\n";

		try
		{
			Toker t;
			t.setMode(Toker::xml);        // comment out to show tokenizing for code
			//    t.setSingleCharTokens("<>");  // will return same results as above for XML

			if (!t.attach(argv[i]))
			{
				std::cout << "\n    can't open file " << argv[i] << "\n\n";
				continue;
			}
			t.returnComments();  // remove this statement to discard comment tokens
			std::string temp;
			do
			{
				temp = t.getTok();
				std::cout << "  ln: " << t.lines() << ", br lev: "
					<< t.braceLevel() << ", tok size: " << std::setw(3) << temp.length() << " -- ";
				if (temp != "\n")
					std::cout << temp << std::endl;
				else
					std::cout << "newline\n";
			} while (temp != "");
		}
		catch (std::exception& ex)
		{
			std::cout << "  " << ex.what() << "\n\n";
		}
	}
}