C++ TokenStream::NextToken方法代码示例

/**
 * A convenience method used for parsing out tokens, this is kind of a repetitive
 * task so it has been written once here.
 *
 * @param ts The token stream to draw from.
 * @param token The token as string to match.
 * @return true if the token was matched, false otherwise.
 */
bool Case_::MatchToken_( TokenStream& ts
                       , std::string const& token) const
{
    bool success = ts.HasTokens();
    if (success)
    {
        Element currentElement = ts.NextToken();
        int const TYPE = currentElement.Type();
        std::string const VALUE(currentElement.ToString());
        success = (TYPE == Types::TOKEN && VALUE == token);
    }
    return success;
}

/**
 * Parse the end of the token stream.
 *
 * @param stream -- Stream that has been prepped.
 */
bool Builtin_::ParseEnd_(TokenStream& stream) const
{
    bool is_end = false;

    if (stream.HasTokens())
    {
        stream.Push();
        Element current_element = stream.NextToken();
        is_end =  (current_element.Type() == Types::TOKEN 
               &&  current_element.ToString() == ")" ); 

        stream.Rollback();
    }

    return is_end;
}

/**
 * Parse out any element possible EXCEPT FOR TOKENS!
 *
 * @param in -- THe token stream to parse from.
 * @param element -- The element to populate if possible. 
 *
 * @return true if something was parsed, false otherwise.
 */
bool Parser::ParseAny( TokenStream& in, Element& element ) const
{
    bool success = false;

    if (in.HasTokens())
    {
        in.Push();

        // First rule of thumb, if the first element in the stream is non-token then
        // return that element since it should be the first.
        strine::Element current = in.NextToken();
        if ( current.Type() != Types::TOKEN )
        {
            element = current;
            success = true;
        }
        else
        {
            in.Rollback();
            strine::Element element_out;
    
            // Go through all the parsables in the container, and try to parse them.
            // If it can be parsed, then do it. 
            size_t const PARSABLES_SIZE = this->parsables.size();
            for(size_t i=0; i<PARSABLES_SIZE; ++i)
            {
                std::shared_ptr<ParsableElement_> parsable = this->parsables[i];
    
                success = parsable->Process( *this
                                           , in 
                                           , element_out );
     
                if (success)
                {
                    element = element_out;
                    break;
                }
            }
        }
    }
     
    return success;
}

/**
 * @return true if the token stream in its present state can convert
 * the token stream to a set expression.
 *
 * @param in -- The token stream.
 * @return true if the stream of tokens can be accepted, false otherwise.
 */ 
bool Function_::CanAccept_( Parser const& parser
                     , TokenStream& stream 
                     , std::vector<strine::Element>& elements) const
{
    bool can_accept = false;

    // (
    if (stream.HasTokens())
    {
        Element current_element = stream.NextToken();
        can_accept =  (current_element.Type() == Types::TOKEN)
                   && (current_element.ToString() == "("); 
    }

    // function
    if (can_accept && stream.HasTokens())
    {
        Element current_element = stream.NextToken();
        can_accept =  (current_element.Type() == Types::TOKEN)
                   && (current_element.ToString() == "function"); 

    }

    // (
    if (can_accept && stream.HasTokens())
    {
        Element current_element = stream.NextToken();
        can_accept =  (current_element.Type() == Types::TOKEN)
                   && (current_element.ToString() == "("); 

    }

    // args
    while(can_accept && stream.HasTokens())
    {
        strine::Element current_element = stream.NextToken();
        
        if (current_element.Type() == Types::TOKEN)         
        {
            if (current_element.ToString() == ")")
            {
                break;
            }
            else
            {
                can_accept = false;
            }
        }
        else if (current_element.Type() == Types::VARIABLE)
        {
            elements.push_back(current_element); 
        }
        else
        {
            can_accept = false;
        }
    }

    if (can_accept)
    {
        stream.Push();
        strine::Element new_body;
        can_accept = parser.ParseAny(stream, new_body);

        if (can_accept && stream.HasTokens())
        {
            stream.Consume();
            Element current_element = stream.NextToken();
            can_accept =  (current_element.Type() == Types::TOKEN)
                       && (current_element.ToString() == ")"); 

            elements.push_back(new_body);
        }
        else
        {
            stream.Rollback();
            can_accept = false;
        }
    }

    return can_accept;
}

/**
 * One of the key methods inside of parser can take a general description
 * of what should be on the token stream and attempts to figure it out.
 *
 * @param in -- The input token stream.
 * @param rules -- The set of input rules we tokenize by.
 * @param elements -- The set of elements.
 *
 * @return true if the set of rules can be parsed, false otherwise.
 */
bool Parser::CanParse( TokenStream& in
                     , std::vector<ParseRule> const& rules 
                     , std::vector<Element>& elements) const
{

    bool success = true;
    elements.clear();

    size_t rule_index = 0;
    while(in.HasTokens() && rule_index < rules.size())
    {
        // Grab the current rule.
        ParseRule const& rule = rules[rule_index];

	// Check to see if the current rule states that we expect a token,
        // if this is the case, then try to match the next element as a 
        // token.
        if (rule.Type() == strine::Types::TOKEN)
        {
            strine::Element current_element = in.NextToken();
            std::string const rule_token(rule.Token());
            std::string const element_string(current_element.ToString());

            success = (rule_token == element_string);
        }

        // If the rule states that the type should be ANY, try to parse 
        // out any element by using ParseAny.
        else if (rule.Type() == ParseRule::ANY)
        {
            strine::Element current_element;

            // Push a marker onto the token stream.
            in.Push();

            success = ParseAny( in, current_element );
     
            if (success)
            {
                elements.push_back(current_element);
            }
            else
            {
                in.Rollback(); 
            }
        }

        // If the rule.Type() hasn't been specified, then go ahead and try 
        // to match up the types.
        else 
        {
            strine::Element current_element = in.NextToken();
            if (rule.Type() == current_element.Type())
            {
                success = true;
                elements.push_back(current_element); 
            }
            else
            {
                success = false;
            }
        }

        if (false == success)
        {
            break;
        }

        rule_index += 1;
    }  

    if (rule_index < rules.size())
    {
        success = false;
    }

    return success;
}

/**
 * @return true if the token stream in its present state can convert
 * the token stream to a builtin expression.
 *
 * @param in -- The token stream.
 * @return true if the stream of tokens can be accepted, false otherwise.
 */ 
bool Builtin_::CanAccept_( Parser const& parser
                         , TokenStream& stream 
                         , std::vector<Element>& elements) const
{
    bool can_accept = false;

    // (
    if (stream.HasTokens())
    {
        Element current_element = stream.NextToken();
        can_accept =  (current_element.Type() == Types::TOKEN) 
                   && (current_element.ToString() == "("); 
    }
    can_accept = can_accept && stream.HasTokens();

    // builtin
    if (can_accept)
    {
        Element current_element = stream.NextToken();
        can_accept =  (current_element.Type() == Types::TOKEN)
                   && (current_element.ToString() == "builtin"); 
    }
    can_accept = can_accept && stream.HasTokens();

    // Symbol
    if (can_accept)
    {
        Element current_element = stream.NextToken();
        can_accept =  (current_element.Type() == Types::STRING);
        elements.push_back(current_element);
    }
    can_accept = can_accept && stream.HasTokens();

    // 
    if (can_accept)
    {
        if (ParseEnd_(stream))
        {
            stream.NextToken();
        }
        else
        {
            Element current_element;
            stream.Push();
            can_accept = parser.ParseAny( stream, current_element );
            
            if (false == can_accept)
            {
                stream.Rollback();
            } 
            else
            {
                elements.push_back(current_element);
                stream.Consume();
            }

            can_accept = ParseEnd_(stream);
            if (can_accept)
            {
                stream.NextToken();
            }
        }
    }

    return can_accept;
}

int _tmain(int argc, _TCHAR* argv[])
{
	Initialize_ProductionTable();

	//initializing the predefine constants
	Initializing_SimbolTable();

	std::cout<<"Insert the mathematic expression:\n";

	//input will be my InputStream
	std::string input;
	std::getline( std::cin, input);
	
	//lets pass the lexer over the input
	Lexer* plex = new Lexer(input);
	TokenStream* ptokstrm = new TokenStream();
	try
		{
	ptokstrm = plex->Analize();
		}
	catch( UnExpextedSimbol e )
		{
		std::cout<<"\n\n"<< e.what();
		::getch();
		exit(0);
		}

	#pragma region Lexer Status, in Case of debugging
	#ifdef DEBUG
		std::cout<<"\nLEXER STATUS\n~~~~~~~~~~~~";
		std::cout<<"\n-----------------------------------------------------";
		std::cout<<"\n| The TokenStream\n|\n";
		
		Token tmp_token;
		ptokstrm->GoFirst();
		std::cout<<"| ";
		while( !ptokstrm->EndToken() )
			{
			std::cout<<ptokstrm->GetCurrent();
			ptokstrm->NextToken();
			}

		std::cout<<"\n|\n-----------------------------------------------------";
		std::cout<<"\n| The Token Atributes\n|\n";
		ptokstrm->GoFirst();

		std::cout<<"|    Tag\tPosition in Simbol Table\n";
		std::cout<<"|    ~~~\t~~~~~~~~~~~~~~~~~~~~~~~~\n";
		while( !ptokstrm->EndToken() )
			{	
			//I want the index position of GetCurrent().data but it does't let
			//me do iterator arithmetics, so I do this stinky thing
			if( ptokstrm->GetCurrent().data != SimbolTable.end() )
				{
			int tmp_cont=0;
			stdext::hash_map<std::string,double>::iterator tmp_it = SimbolTable.begin();
			while( tmp_it != ptokstrm->GetCurrent().data )
				{
				++tmp_cont;
				++tmp_it;
				}
			std::cout<<"|    "<<ptokstrm->GetCurrent()/*Hete it is the tagm but it is the same to show the token because of the operator <<*/<<
				"\t\t"<< tmp_cont<<'\n';
			ptokstrm->NextToken();
				}else{//If the position of the table is endm so I say NULL
			std::cout<<"|    "<<ptokstrm->GetCurrent()/*Hete it is the tagm but it is the same to show the token because of the operator <<*/<<
				"\t\t"<< "NULL"<<'\n';
			ptokstrm->NextToken();
				}
			}


		std::cout<<"|\n-----------------------------------------------------";
		std::cout<<"\n| The Simbol Table\n|\n";
		//std::copy(SimbolTable.begin(),SimbolTable.end(),std::ostream_iterator<std::pair<std::string,double>>(std::cout));
		std::cout<<"|      Lexeme\t      Value\n";
		std::cout<<"|      ~~~~~~\t      ~~~~~\n";
		stdext::hash_map<std::string,double>::iterator it = SimbolTable.begin();
		int index=0;
		while( it != SimbolTable.end() )
			{
			std::cout<<"| "<<index<<")     "<<*it;
			++index;
			++it;
			}
		std::cout<<"| "<<"NULL)\n";

		std::cout<<"|\n-----------------------------------------------------";
		//let's put it in the begining again
		ptokstrm->GoFirst();
	#endif
	#pragma endregion

	Parser* ppar = new Parser( *ptokstrm );
	Dash_Structures::DBTree<Token>* pSyntaxTree;
	try
		{
	pSyntaxTree = ppar->Analize();
		}
	catch( BadSintax e )
//.........这里部分代码省略.........