C++ CTokenizer类代码示例

本文整理汇总了C++中CTokenizer类的典型用法代码示例。如果您正苦于以下问题：C++ CTokenizer类的具体用法？C++ CTokenizer怎么用？C++ CTokenizer使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。

在下文中一共展示了CTokenizer类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C++代码示例。

示例1: CByteTokenizer

//
// Load()
// Parses a .vmt file.  Note, the parser is very loose.  .vmt files vary
// so much in the official resources that it is hard to know what is legal.
//
vlBool CVMTFile::Load(IO::Readers::IReader *Reader)
{
	delete this->Root;
	this->Root = 0;

	if(!Reader->Open())
		return vlFalse;

	CByteTokenizer ByteTokenizer = CByteTokenizer(Reader);
	CTokenizer Tokenizer = CTokenizer(&ByteTokenizer);
	CParser Parser = CParser(&Tokenizer);

	try
	{
		this->Root = Parser.Parse();
	}
	catch(char *cErrorMessage)
	{
		LastError.SetFormatted("Error parsing material on line %u (%s).", Tokenizer.GetLine(), cErrorMessage);
	}

	Reader->Close();

	return this->Root != 0;
}

开发者ID:azuisleet，项目名称:VTFLib，代码行数:30，代码来源:VMTFile.cpp

示例2: fopen

CParticleBehaviour CParticleSystemManager::ReadParticleBehaviour( std::string szEmitterScript, std::string szParticleBehaviourName )
{
	FILE *pFile = fopen( szEmitterScript.c_str(), "rt" );
	if ( !pFile )
		return CParticleBehaviour( "NULL", 0.0f, 0.0f );

	CTokenizer tokenizer;
	std::vector<std::string> tokens;

	std::string in = ReadLine( pFile );
	bool bInBehaviour = false;

	float fEffect, fVelocity;
	fEffect = fVelocity = 0.0f;

	while ( in != "<<<EOF>>>" )
	{
		tokens = tokenizer.GetTokens( in, " ,;:\"" );
		if ( tokens.size() == 0 )
		{
			in = ReadLine( pFile );
			bInBehaviour = false;
			continue;
		}

		if ( tokens[0] == "[emitter]" )
			bInBehaviour = false;
		else if ( tokens[0] == "[particle]" )
			bInBehaviour = false;
		else if ( tokens[0] == "[behaviour]" )
			bInBehaviour = true;

		if ( bInBehaviour )
		{
			if ( tokens[0] == "name" && tokens[2] == szParticleBehaviourName )
			{
				// Read until the next blank line
				in = ReadLine( pFile );
				while ( in != "" && in != "<<<EOF>>>" )
				{
					tokens = tokenizer.GetTokens(in);
					if ( tokens[0] == "velocity" )
						fVelocity = (float)atof( tokens[2].c_str() );
					else if ( tokens[0] == "move" )
						fEffect = (float)atof( tokens[2].c_str() );
					in = ReadLine( pFile );
				}
				fclose( pFile );
				return CParticleBehaviour( szParticleBehaviourName, fEffect, fVelocity );
			}
		}

		in = ReadLine( pFile );
	}
	fclose( pFile );

	return CParticleBehaviour( "NULL", 0.0f, 0.0f );
}

开发者ID:TimToxopeus，项目名称:grapplon2，代码行数:58，代码来源:ParticleSystemManager.cpp

示例3: ParseVelSetState

void CStateParser::ParseVelSetState(CTokenizer &tok,CStateManager &StateManager)
{
	VELSET *temp=(VELSET*) m_pAlloc->Alloc(sizeof(VELSET));
	while( !tok.CheckToken("[", false) && !tok.AtEndOfFile() )
	{
		StateManager.NewInst();
		if( tok.CheckToken("y") )
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok);  

			EvaluateExpression(tok,StateManager);
			temp->common.vely = StateManager.GetParamIns();

		}else if( tok.CheckToken("x") )
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager);  
			temp->common.velx = StateManager.GetParamIns();
		}else if (tok.CheckToken("persistent") ) 
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager); 
		}
	}

	StateManager.SetController(temp);
	StateManager.NewInst(); 
}

开发者ID:oraclehan，项目名称:mutest，代码行数:33，代码来源:StateParser.cpp

示例4: ParseChangeState

void CStateParser::ParseChangeState(CTokenizer &tok,CStateManager &StateManager)
{
	CHANGESTATE *temp=(CHANGESTATE*) m_pAlloc->Alloc(sizeof(CHANGESTATE));
	//TODO:Check for Required parameters and print error msg
	while( !tok.CheckToken("[", false) && !tok.AtEndOfFile() )
	{
		StateManager.NewInst();
		if( tok.CheckToken("value") )
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok);  
                          
			EvaluateExpression(tok,StateManager);
			temp->common.value = StateManager.GetParamIns();
		}else if( tok.CheckToken("ctrl") )
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager);  
            temp->common.ctrl = StateManager.GetParamIns();      
		}else if ( tok.CheckToken("anim") )
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok); 
                  
			EvaluateExpression(tok,StateManager); 
			temp->common.anim = StateManager.GetParamIns();    
                  
		}            
	}
	StateManager.SetController(temp);
	StateManager.NewInst(); 
}

开发者ID:oraclehan，项目名称:mutest，代码行数:34，代码来源:StateParser.cpp

示例5: EvaluateExpression

//Generates the Opcode sequenz for the trigger statement
//evaluates and + and - expression
void CStateParser::EvaluateExpression(CTokenizer &tok,CStateManager &StateManager)
{
      
     Term(tok,StateManager);
     
     while( tok.CheckToken("+",false) || tok.CheckToken("-",false) ||
			tok.CheckToken("&&",false) || tok.CheckToken("||",false) 
		 && !tok.AtEndOfLine() )
     {  
        
    
        if( tok.CheckToken("+") )
        {
            Term(tok,StateManager);
            StateManager.AddInstruction(OP_ADD,0,"#");
           
        }
        
        if( tok.CheckToken("-") )
        {
           Term(tok,StateManager);
           StateManager.AddInstruction(OP_SUB,0,"#");
             
        }

		if( tok.CheckToken("&&") )
		{
			Term(tok,StateManager);
			StateManager.AddInstruction(OP_AND,0,"#");

		}

		if( tok.CheckToken("||") )
		{
			Term(tok,StateManager);
			StateManager.AddInstruction(OP_OR,0,"#");

		}
  
     }
}

开发者ID:oraclehan，项目名称:mutest，代码行数:43，代码来源:StateParser.cpp

示例6: Term

//evaluates and * and / expression
void CStateParser::Term(CTokenizer &tok,CStateManager &StateManager)
{
         
     Primary(tok,StateManager);
     
     //search for operators 
     while( tok.CheckToken("*",false) || tok.CheckToken("/",false)  || 
            tok.CheckToken("=",false) || tok.CheckToken("!=",false) ||
            tok.CheckToken("<",false) || tok.CheckToken("<=",false) ||
            tok.CheckToken(">",false) || tok.CheckToken(">=",false) ||
            tok.CheckToken(":",false) || /*tok.CheckToken("&&",false) || 
            tok.CheckToken("||",false)||*/ tok.CheckToken("^^",false) ||
            tok.CheckToken("&",false) || tok.CheckToken("~",false)  ||
            tok.CheckToken("|",false) || tok.CheckToken("^",false)  || 
            tok.CheckToken("%",false) || tok.CheckToken("(", false) 	
			&& !tok.AtEndOfLine() )  
     {  
      
            if( tok.CheckToken("*") )
            {
                //Have we a ** operator?
                if( tok.CheckToken("*") )
                {
                    Primary(tok,StateManager);
                    StateManager.AddInstruction(OP_SQUARE,0,"#");
                    
                }else
                {
                    Primary(tok,StateManager);
                    StateManager.AddInstruction(OP_MUL,0,"#");
                }
            }
         
           if( tok.CheckToken("/") )
           {
               Primary(tok,StateManager);
               StateManager.AddInstruction(OP_DIV,0,"#");
           }
           
           if( tok.CheckToken("=") )
           {    
                //check for intervall operator
                if( tok.CheckToken("(") )
                {
                    //evaluate first expression                   
                    EvaluateExpression(tok,StateManager);
                    if( !tok.CheckToken(",") )
                        Error("Expectetd a , in intervall operator",tok);

                    //evaluate second expression 
                    EvaluateExpression(tok,StateManager);
                    
                    //intervall op =(,)
                    if( tok.CheckToken(")") )
                    {
                        StateManager.AddInstruction(OP_INTERVALOP4,0,"#");
                    }else if( tok.CheckToken("]") )
                    {
                         StateManager.AddInstruction(OP_INTERVALOP3,0,"#");
                    }
                    
                }else if (tok.CheckToken("[") )
                {
                    //evaluate first expression                   
                    EvaluateExpression(tok,StateManager);
                    if( !tok.CheckToken(",") )
                        Error("Expectetd a , in intervall operator",tok);

                    //evaluate second expression 
                    EvaluateExpression(tok,StateManager);
                    
                    //intervall op =[,)
                    if( tok.CheckToken(")") )
                    {
                       StateManager.AddInstruction(OP_INTERVALOP2,0,"#");
                    }else if( tok.CheckToken("]") )
                    {
                         StateManager.AddInstruction(OP_INTERVALOP1,0,"#");
                    }
                      
                }
                else // is the = op
                {                
					//evalute the right side of the operator
					Primary(tok,StateManager);
					StateManager.AddInstruction(OP_EQUAL,0,"#");
                }
           }
           
           if( tok.CheckToken(":") )
           {    //evalute the right side of the operator
                if(tok.CheckToken("="))
                {
                 EvaluateExpression(tok,StateManager);
                 PrintMessage("TODO:Handel assign operator :=");
                }
           }
           
           if( tok.CheckToken("!=") )
//.........这里部分代码省略.........

开发者ID:oraclehan，项目名称:mutest，代码行数:101，代码来源:StateParser.cpp

示例7: PareseState

void CStateParser::PareseState(CTokenizer &tok,CStateManager &StateManager)
{
    std::string ret;     
	while( !tok.CheckToken("[",false) && !tok.AtEndOfFile() )
	{
		if( tok.CheckToken("type") )
		{
		   if( !tok.CheckToken("=") )
			   throw(CError("expected ="));
           
		   ret = tok.GetToken();
		   nController=GetControllerType(ret.c_str(),tok );
      
        
		} else if ( tok.CheckTokenLike("trigger", false) )
		{
			std::string trigger = tok.GetToken();

			if( !tok.CheckToken("=") )
			   Error("expected =",tok);
         
			//fix me add triggerall
		
			ParseTrigger(tok,StateManager);
			if (trigger.compare("triggerall") == 0)
			{
				StateManager.AddTriggerToState(TRIGGERALL);
			}
			else
			{
				//½âÎö³öÊÇÄÄ¸ötrigger
				std::string value = trigger.substr(strlen("trigger"), trigger.length() - strlen("trigger"));
				StateManager.AddTriggerToState(atoi(value.c_str()));
			}

		}/*else  if( tok.CheckToken("trigger1") 
					||  tok.CheckToken("trigger2")
					||  tok.CheckToken("trigger3")
					||  tok.CheckToken("trigger4")
					||  tok.CheckToken("trigger5")
					||  tok.CheckToken("trigger6")
					||  tok.CheckToken("trigger7")
					||  tok.CheckToken("trigger8")
					||  tok.CheckToken("trigger9")
					||  tok.CheckToken("trigger10")
					||  tok.CheckToken("trigger11")
					||  tok.CheckToken("trigger12")
					||  tok.CheckToken("trigger13")
					||  tok.CheckToken("trigger14")
					||  tok.CheckToken("trigger15")
					||  tok.CheckToken("trigger16")
					||  tok.CheckToken("trigger17")
					||  tok.CheckToken("trigger18")
					||  tok.CheckToken("trigger19")
					||  tok.CheckToken("trigger20"))        
		{
			if( !tok.CheckToken("=") )
				Error("expected =",tok);
			ParseTrigger(tok,StateManager);
			StateManager.AddTriggerToState(TRIGGERNUM);
		}*/
		else break;
    

       
	}
	//parse the controller
	ParserController(tok,StateManager,nController); 
	StateManager.AddTypeToState(nController);
}

开发者ID:oraclehan，项目名称:mutest，代码行数:70，代码来源:StateParser.cpp

示例8: ParseTrigger

void CStateParser::ParseTrigger(CTokenizer &tok,CStateManager &StateManager)
{
    tok.SetReturnNegativeSeperatelyFromNumber(true);
    EvaluateExpression(tok,StateManager);
    tok.SetReturnNegativeSeperatelyFromNumber(false);
}

开发者ID:oraclehan，项目名称:mutest，代码行数:6，代码来源:StateParser.cpp

示例9: ParseStateFile

void CStateParser::ParseStateFile(const char* strFileName,CStateManager &StateManager,CAllocater *a)
{
     //Set pointer to allocater
     m_pAlloc=a;
     
     CTokenizer tok;
     bool foundState=false;
     
     if( !tok.OpenFile(strFileName) )
        throw(CError("CStateParser::ParseState: File %s not found",strFileName));
        
     tok.SetIsCaseSensitive(false);
     tok.SetReturnNegativeSeperatelyFromNumber(false);
        

     while( !tok.AtEndOfFile() )
     {
          foundState=false;
          
          if( tok.CheckToken("[") )
          {
            
             if( tok.CheckToken("statedef") )
             {
                 foundState=true;
                 if(!tok.CheckTokenIsNumber())
                     Error("Expected a number in statedef block",tok);
                     
                 StateManager.AddStateDef(tok.GetInt());
                
                 //Skip useless stuff
                 while( !tok.AtEndOfLine() )
                         tok.GetToken();
                
                //parse the state def
                ParseStateDef(tok,StateManager);        
                         
                         
             }
             if( tok.CheckToken("state") )
             {
               foundState=true;  
               
               if(!tok.CheckTokenIsNumber())
                     Error("Expected a number in state block",tok);
                 
			   int stateno = tok.GetInt();

			   tok.CheckToken(",");

// 			   if(!tok.CheckTokenIsQuotedString()
// 				   && !tok.CheckTokenIsNumber())
// 				   Error("Expected a number in state block",tok);

			   std::string str = tok.GetToken();
               StateManager.AddState(stateno, str.c_str());
                                  
			   while(!tok.AtEndOfLine())
				   tok.GetToken();  
                         
				PareseState(tok,StateManager);
                   
             }
    
          
          }
          
          //skip useless stuff
          if(!foundState)
             tok.GetToken(); 
            
     }
     
     tok.CloseFile();
}

开发者ID:oraclehan，项目名称:mutest，代码行数:75，代码来源:StateParser.cpp

示例10: ParseStateDef

void CStateParser::ParseStateDef(CTokenizer &tok,CStateManager &StateManager)
{
	while( !tok.CheckToken("[",false) && !tok.AtEndOfFile() )
	{
		//parse state type
		if( tok.CheckToken("type") )
		{
			if( !tok.CheckToken("=") )
			Error("expected =",tok);
         
			//to get a single char   
			char c=tok.GetToken()[0];
			//make sure we use uperchars
			if(c >= 97)
			c-=32;
            
                  
			switch(c)
			{
			case 'S':
				StateManager.SetStateDefType(stand); 
			break;
            
			case 'C':
				StateManager.SetStateDefType(crouch);     
			break;
                        
			case 'A':
				StateManager.SetStateDefType(air); 
			break;
            
			case 'L':
				StateManager.SetStateDefType(liedown);     
			break;
            
			case 'U':
				StateManager.SetStateDefType(untouch);    
			break;
            
			default:
				Error("Unknown statetype",tok);
			break;
                      
                  
			}

		} else if( tok.CheckToken("movetype") )
		{
			if( !tok.CheckToken("=") )
			Error("expected '=' in line ",tok);
            
			//to get a single char   
			char c=tok.GetToken()[0];
			//make sure we use uperchars
			if(c >= 97)
			c-=32;
           
			switch(c)
			{
			case 'A':
				StateManager.SetStateMoveType(attack);
			break;

			case 'I':
				StateManager.SetStateMoveType(idle);
			break;

			case 'H':
				StateManager.SetStateMoveType(hit);
			break;

			case 'U':
				StateManager.SetStateMoveType(untouch);
			break;
            
			default:
				Error("Unknown movetype",tok);
			break;
            
                  
                  
			}      
           
              
         
		} else if( tok.CheckToken("physics") )
		{
			if( !tok.CheckToken("=") )
			Error("expected =",tok);
         
			//to get a single char   
			char c=tok.GetToken()[0];
			//make sure we use uperchars
			if(c >= 97)
			c-=32;
            
                  
			switch(c)
			{
			case 'S':
//.........这里部分代码省略.........

开发者ID:oraclehan，项目名称:mutest，代码行数:101，代码来源:StateParser.cpp

示例11: ParseVarSet

void CStateParser::ParseVarSet( CTokenizer &tok, CStateManager &StateManager )
{
	VARSET *temp=(VARSET*) m_pAlloc->Alloc(sizeof(VARSET));
	//TODO:Check for Required parameters and print error msg
	while( !tok.CheckToken("[", false) && !tok.AtEndOfFile() )
	{
		StateManager.NewInst();
		if( tok.CheckToken("sysvar") )
		{
			if( !tok.CheckToken("(") )
				Error("expected (",tok);  

			int value = tok.GetInt();

			StateManager.AddInstruction(OP_PUSH, value, "#");

			if( !tok.CheckToken(")") )
				Error("expected )",tok);

			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager);
			temp->common.sysvar = StateManager.GetParamIns();
		}else if( tok.CheckToken("sysfvar") )
		{
			if( !tok.CheckToken("(") )
				Error("expected (",tok);  

			int value = tok.GetInt();

			StateManager.AddInstruction(OP_PUSH, value, "#");

			if( !tok.CheckToken(")") )
				Error("expected )",tok);

			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager);
			temp->common.sysfvar = StateManager.GetParamIns();     
		}else if ( tok.CheckToken("var") )
		{
			if( !tok.CheckToken("(") )
				Error("expected (",tok);  

			int value = tok.GetInt();

			StateManager.AddInstruction(OP_PUSH, value, "#");

			if( !tok.CheckToken(")") )
				Error("expected )",tok);

			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager);
			temp->common.var = StateManager.GetParamIns();     

		}else if ( tok.CheckToken("fvar") )
		{
			if( !tok.CheckToken("(") )
				Error("expected (",tok);  

			int value = tok.GetInt();

			StateManager.AddInstruction(OP_PUSH, value, "#");

			if( !tok.CheckToken(")") )
				Error("expected )",tok);

			if( !tok.CheckToken("=") )
				Error("expected =",tok);

			EvaluateExpression(tok,StateManager);
			temp->common.fvar = StateManager.GetParamIns();     

		}             
	}
	StateManager.SetController(temp);
	StateManager.NewInst(); 
}

开发者ID:oraclehan，项目名称:mutest，代码行数:82，代码来源:StateParser.cpp

示例12: Error

void CStateParser::Error(const char * strErrorMsg,CTokenizer &tok)
{
     throw(CError("Parser error:\nin file %s at line %i:\n%s",tok.GetFileName()
                                                         ,tok.GetLineNumber()
                                                         ,strErrorMsg));                  
}

开发者ID:oraclehan，项目名称:mutest，代码行数:6，代码来源:StateParser.cpp

示例13: CTokenizer

CTokenizer* CTokenizer::Create(UINT dwFlags)
{
	CTokenizer* theTokenizer = new CTokenizer();
	theTokenizer->Init(dwFlags);
	return theTokenizer;
}

开发者ID:Hasimir，项目名称:jedi-outcast-1，代码行数:6，代码来源:Tokenizer.cpp

示例14: Primary

//evaluates a primary
void CStateParser::Primary(CTokenizer &tok,CStateManager &StateManager)
{
     //a negate operator
     if( tok.CheckToken("-") )
     {
           //EvaluateExpression(tok,StateManager);
           Primary(tok,StateManager);
           StateManager.AddInstruction(OP_NEG,0,"#");  
     }else if( tok.CheckTokenIsNumber() )  //we have a number
     {
         StateManager.AddInstruction(OP_PUSH,tok.GetFloat(),"#");                       
     }else if( tok.CheckTokenIsQuotedString() ) //it is a "quitedstring"
     {
         StateManager.AddInstruction(OP_PUSH,0,tok.GetToken());  
     }else if( tok.CheckToken("(") ) //here we have to check a lot of possibilitys
     {
         EvaluateExpression(tok,StateManager);
         
         if( !tok.CheckToken(")") )
                Error("Missing )",tok);
         
     }else if( tok.CheckToken("!") )
     {
          Primary(tok,StateManager);
          StateManager.AddInstruction(OP_NOT,0,"#");
     }
     else //check for a trigger name
     {
		 std::string ret = tok.GetToken();
         int i=GetTriggerType(ret.c_str(),tok);
                 
		 if ( i == OP_Vel - OP_Abs) 
		 {
			 if (tok.CheckToken("x"))
			 {
				 StateManager.AddInstruction(OP_Vel,1.0,"#");
			 }
			 else if  (tok.CheckToken("y"))
			 {
				 StateManager.AddInstruction(OP_Vel,0.0,"#");
			 }
		 }else if ( i == OP_Pos - OP_Abs) 
		 {
			 if (tok.CheckToken("x"))
			 {
				 StateManager.AddInstruction(OP_Pos,1.0,"#");
			 }
			 else if  (tok.CheckToken("y"))
			 {
				 StateManager.AddInstruction(OP_Pos,0.0,"#");
			 }
		 }
		 else if ( i == OP_Const - OP_Abs) 
		 {
			 if (!tok.CheckToken("("))
				 Error("Missing (",tok);

			 StateManager.AddInstruction(OP_Const,0.0,tok.GetToken());
			 if (!tok.CheckToken(")"))
				 Error("Missing )",tok);
		 }
		 else if ( i == OP_IfElse - OP_Abs) 
		 {
			 if (!tok.CheckToken("("))
				 Error("Missing (",tok);

			 EvaluateExpression(tok,StateManager);
			 if (!tok.CheckToken(","))
				 Error("Missing ,",tok);

			 Term(tok,StateManager);

			 if (!tok.CheckToken(","))
				 Error("Missing ,",tok);

			 Term(tok,StateManager);

			 StateManager.AddInstruction(OP_IfElse,0,"#");
			 if (!tok.CheckToken(")"))
				 Error("Missing )",tok);
		 }
		 else if (i == OP_StateType - OP_Abs)
		 {
			 if (tok.CheckToken("="))
			 {
				 StateManager.AddInstruction(OP_StateType,0,"#");
				 StateManager.AddInstruction(OP_PUSH,0,tok.GetToken());
				 StateManager.AddInstruction(OP_EQUAL,0,"#");
			 }else if (tok.CheckToken("!="))
			 {
				 StateManager.AddInstruction(OP_StateType,0,"#");
				 StateManager.AddInstruction(OP_PUSH,0,tok.GetToken());
				 StateManager.AddInstruction(OP_NOTEQUAL,0,"#");
			 }
			 
		 }
		 else if (i == OP_P2StateType - OP_Abs)
		 {
			 if (tok.CheckToken("="))
//.........这里部分代码省略.........

开发者ID:oraclehan，项目名称:mutest，代码行数:101，代码来源:StateParser.cpp

示例15: test

  // Launch unit test.
  EUnitTestResult test()
  { CALL
    CString test;
    Tuint offsets[] = { 1, 2, 3 };
    CWrappedArray<Tuint> offsetarray(offsets);
    CTokenizerFunctionOffsetDelimiters tf;
    CTokenizer tokenizer;
    CTokenizer::TIterator it;

    // Prepare tokenizer with default offset tokenizer function (initialized with a single offset value (1)).
    UT_ASSERT(tokenizer.setTokenizerFunction(tf));
    // Perform testing.
    test = STR("1234567890");
    UT_ASSERT(tokenizer.setInputContainer(test));
    it = tokenizer.getItTop();
    UT_ASSERT_EQUAL(*it, CString(STR("1")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("2")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("3")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("4")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("5")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("6")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("7")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("8")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("9")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("0")));
    UT_ASSERT_EQUAL(it.stepForward(), 0);

    // Prepare tokenizer with offset tokenizer function which wraps offsets and returns last partially parser token.
    UT_ASSERT(tf.set(offsetarray.getItFirst(), true, true));
    UT_ASSERT(tokenizer.setTokenizerFunction(tf));
    // Perform testing.
    test = STR("1234567890ABCDEF");
    UT_ASSERT(tokenizer.setInputContainer(test));
    it = tokenizer.getItTop();
    UT_ASSERT_EQUAL(*it, CString(STR("1")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("23")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("456")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("7")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("89")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("0AB")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("C")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("DE")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("F")));
    UT_ASSERT_EQUAL(it.stepForward(), 0);

    // Prepare tokenizer with offset tokenizer function which wraps offsets but does not return last partially parser token.
    UT_ASSERT(tf.set(offsetarray.getItFirst(), true, false));
    UT_ASSERT(tokenizer.setTokenizerFunction(tf));
    // Perform testing.
    test = STR("1234567890ABCDEF");
    UT_ASSERT(tokenizer.setInputContainer(test));
    it = tokenizer.getItTop();
    UT_ASSERT_EQUAL(*it, CString(STR("1")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("23")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("456")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("7")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("89")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("0AB")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("C")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("DE")));
    UT_ASSERT_EQUAL(it.stepForward(), 0);

    // Prepare tokenizer with offset tokenizer function which does not wrap offsets and return last partially parser token.
    UT_ASSERT(tf.set(true, offsetarray, false, false));
    UT_ASSERT(tokenizer.setTokenizerFunction(tf));
    // Perform testing.
    test = STR("1234567890ABCDEF");
    UT_ASSERT(tokenizer.setInputContainer(test));
    it = tokenizer.getItTop();
    UT_ASSERT_EQUAL(*it, CString(STR("1")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("23")));
    UT_ASSERT_EQUAL(it.stepForward(), 1);
    UT_ASSERT_EQUAL(*it, CString(STR("456")));
    UT_ASSERT_EQUAL(it.stepForward(), 0);
//.........这里部分代码省略.........

开发者ID:pombredanne，项目名称:Depth，代码行数:101，代码来源:test-tokenizer-CTokenizerOffsetDelimiters.cpp

注：本文中的CTokenizer类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。