C++ TLex::MarkedToken方法代码示例

TPtrC NextWord(TLex& aLex)
	{
	aLex.SkipSpaceAndMark();
	aLex.SkipCharacters();
	TPtrC word(aLex.MarkedToken());
	return word;
	}

TPtrC LineRemainder(TLex& aLex)
	{
	aLex.Mark();
	if (SkipTo(aLex, KNewLine) == KErrNone)
		{
		return aLex.MarkedToken();
		}
	else
		{
		return aLex.Remainder();
		}
	}

TPtrC TfrLex::GetL( TLex& aLex, const TChar  aChar )
{
// Skip spaces and mark the token's start point.
aLex.SkipSpace();
TLexMark mark;

aLex.Mark(mark);    

if (aLex.Peek() == '"' )
	{
	// Skip the " and do find next " followed by eos, space or aChar.
	aLex.Inc();
	TChar x;
	while ( x = aLex.Get(), !x.Eos() )
		{
		if ( x == '"' )
			{
			// Found a " character - but is it the end of the token?
			x = aLex.Peek(); // peek the next character
			if ( x.Eos() || x.IsSpace() || x == aChar )
				// End of token: return token.
				return aLex.MarkedToken(mark);
			}
		}
	// Unget and L E A V E because did not find the end " of token.
	aLex.UnGetToMark(mark);
	User::Leave(KErrArgument);
	return aLex.MarkedToken(mark); // never reached (l e a v e).
	}
else
	{
	// Is not a "*" token: find eos or the next space or the aChar
	// and return the token.
	TChar x;
	while ( x = aLex.Peek(), !x.Eos() && !x.IsSpace() && x != aChar )
		aLex.Inc();

	return aLex.MarkedToken(mark);
	}
}

void CScriptFile::FoundNewItemL(const TDesC& aText, TLex& arInput, TInt& arCurrentItemStart, CScriptSection& aSection, CScriptSectionItem*& arCurrentItem)
	{
	TPtrC token = arInput.MarkedToken();

	ParseAndSetItemValueL(aText, arInput, arCurrentItemStart, arCurrentItem);

	arInput.SkipSpaceAndMark();
	arCurrentItemStart = arInput.Offset();

	TPtrC itemEnd(KScriptItemEnd);
	const TInt length = token.Length() - itemEnd.Length();
	arCurrentItem = &aSection.ReplaceItemL(token.Left(length), KNullDesC);
	}

TPtrC NextCommand(TLex& aLex)
	{
	TPtrC command;
	if (SkipToNextCommand(aLex) == KErrNone)
		{
		SkipToEnd(aLex, KPartialCommandStart);
		aLex.Mark();
		aLex.SkipCharacters();
		command.Set(aLex.MarkedToken());
		}
	else
		{
		aLex.Inc(aLex.Remainder().Length());
		}
	return command;
	}

// -----------------------------------------------------------------------------
// CAknKeyRotatorImpl::SkipSpaces
// Goes over digits and returns that sequence.
// -----------------------------------------------------------------------------
//
TPtrC CAknKeyRotatorImpl::GetDigits( TLex& aLex )
    {
    // Mark current place and go over digits.
    aLex.Mark();
    while ( !aLex.Eos() && !IsEndOfLine( aLex.Peek() ) )
        {
        if ( aLex.Peek().IsDigit() )
            {
            aLex.Inc();
            }
        else
            {
            break;
            }
        }
    return aLex.MarkedToken();
    }

EXPORT_C TInt CScriptFile::GetNextWord(TLex& aInput, TChar aDelimiter, TPtrC& aOutput)
	{
	//Get to the start of the descriptor
	while (!aInput.Peek().IsAlphaDigit() && aInput.Peek() != '+' && !aInput.Eos() && aInput.Peek() != aDelimiter)
		aInput.Inc();

	if (aInput.Eos())
		return KErrNotFound;

	aInput.Mark();

	while (aInput.Peek() != aDelimiter && !aInput.Eos())
		aInput.Inc();

	aOutput.Set(aInput.MarkedToken());

	if (!aInput.Eos())
		aInput.SkipAndMark(1);

	return KErrNone;
	}

TPtrC TextToNextCommand(TLex& aLex)
	{
	aLex.Mark();

	if (SkipToNextCommand(aLex) != KErrNone)
		{
		aLex.Inc(aLex.Remainder().Length());
		}

	TLex lex(aLex.MarkedToken());
	
	// Strip leading white space.
	while (!lex.Eos())
		{
		if (lex.Peek().IsSpace())
			{
			lex.Get();
			}
		else
			{
			break;
			}
		}
	lex.Mark();
	TInt startOffset = lex.Offset();

	// Remove trailing white space.
	lex.Inc(lex.Remainder().Length());
	while (lex.Offset() > startOffset)
		{
		lex.UnGet();
		if (!lex.Peek().IsSpace())
			{
			lex.Get();
			break;
			}
		}

	return lex.MarkedToken();
	}

//
//  ParseMessageL()
//
//  Simple Notification:
//      Looks for number of messages and "From:" field
//      writes numMsg into the parsed field array, sets
//      entry.iDetails to the "From:" 
//  Complicated Notification:
//      does the above and writes everything else into
//      the array....
//
void CEmailNotificationParser::ParseMessageL()
    {
    TBool isSimpleNotification = EFalse; 
    TLex  tokenLex;
    TPtrC fieldName;
    TPtrC fieldValue;

	// reset parsedfield array
	for(TInt i = iParsedFieldArray->Count();--i>=0;)
		(*iParsedFieldArray)[i]->SetFieldValueL(_L(""));

    // Set extraction mark at first character
    iSms.SkipSpaceAndMark();
	if(iSms.Peek() == KCharSlash)
		{
		// Check first line is <header>
		iSms.SkipCharacters();

		if (iSms.MarkedToken() != KEmailHeader)
			{
			User::Leave(KBspInvalidMessage);
			}
		// Get <new-amount> from second line and check for terminating linefeed
		iSms.SkipSpaceAndMark();
		}

    // Val() seeks forward from next char position, looking for valid
    // digits and incrementing next char as it goes.
    if (!(iSms.Val(iMessageCount) == KErrNone  &&  iMessageCount >= 0))     // If marked token is not a valid positive integer
        {
        iMessageCount = 0;
        User::Leave(KBspInvalidMessage);
        }
    else
        {
        fieldValue.Set(iSms.MarkedToken());                            //  The message count..
        AddParsedFieldL(KHeaderNumberMessages, fieldValue, ETrue);
        }

    // Next character may now be at newline or space after integer.
    // If at space, advance to newline.
    while (iSms.Peek() != KCharLineFeed && !iSms.Eos())
        iSms.Inc();

    iSms.SkipSpaceAndMark();

    // Now parse the rest of the fields, if any.
    while (!iSms.Eos())
        {
        while (iSms.Peek() != KCharLineFeed  &&  !iSms.Eos())
            iSms.Inc();     //  Skip to next delimiter

        if (iSms.Eos())
            break;          //  we've finished break out of the function

        if (iSms.TokenLength() == 0)
            User::Leave(KBspSmartMessageInvalidToken);

        //  Parsing....
        tokenLex.Assign(iSms.MarkedToken());        // Assign token to a new TLex
        while (tokenLex.Peek() != KCharColon  &&  !tokenLex.Eos())
            tokenLex.Inc();     //  Advance to a ':'

        if (tokenLex.Eos()  ||  tokenLex.TokenLength() == 0)
            User::Leave(KBspSmartMessageInvalidToken);


        fieldName.Set(tokenLex.MarkedToken());      // Store (pointer to) field name
        tokenLex.Inc();

		//fix for DEF017686
		LeaveIfEmptyFieldsL(fieldName,tokenLex);
		
        
        tokenLex.SkipSpaceAndMark();                // Step past optional spaces

		//fix for DEF017686
        LeaveIfEmptyFieldsL(fieldName,tokenLex);
	
		// if it's the server id field try to extract the id value
		// and match to an existing email service
		if(fieldName.CompareF(KHeaderServerId)==KErrNone)
			{
			TInt valErr = tokenLex.Val(iServerId);
			if(valErr != KErrNone)
				iServerId = 0;
			else
				GetEmailServicesL();

//.........这里部分代码省略.........

void CCommandInfoFile::ReadDetailsL(TLex& aLex, RFs& aFs, const TDesC& aFileName)
	{
	SkipToNextCommand(aLex); // Ignore everything before the first '==' command.

	while (!aLex.Eos())
		{
		TLexMark mark;
		aLex.Mark(mark);
		TPtrC command(NextCommand(aLex));
		if (command == KNullDesC)
			{
			// Do nothing - we're at the end of the string.
			}
		else if (command == KCmndName)
			{
			aLex.SkipSpaceAndMark();
			aLex.SkipCharacters();
			iName.Set(aLex.MarkedToken());
			}
		else if (command == KCmndShortDescription)
			{
			iShortDescription.Set(TextToNextCommand(aLex));
			}
		else if (command == KCmndLongDescription)
			{
			iLongDescription.Set(TextToNextCommand(aLex));
			}
		else if (command == KCmndSeeAlso)
			{
			iSeeAlso.Set(TextToNextCommand(aLex));
			}
		else if (command == KCmndCopyright)
			{
			iCopyright.Set(TextToNextCommand(aLex));
			}
		else if (command == KCmndSmokeTest)
			{
			// Hmm no easy way to get the line number we're currently on
			iSmokeTestLineNumber = 1;
			TLex lex(aLex);
			lex.Inc(-aLex.Offset()); // Only way to put a TLex back to the beginning!
			TPtrC preceding = lex.Remainder().Left(aLex.Offset());
			const TUint16* ptr = preceding.Ptr();
			const TUint16* end = ptr + preceding.Length();
			while (ptr != end)
				{
				if (*ptr++ == '\n') iSmokeTestLineNumber++;
				}
			// At this point iSmokeTestLineNumber points to the "==smoketest" line - add 2 to skip this line and the blank line below it
			iSmokeTestLineNumber += 2;
			iSmokeTest.Set(TextToNextCommand(aLex));
			}
		else if (command == KCmndArgument)
			{
			ReadArgumentL(aLex, aFileName);
			}
		else if (command == KCmndOption)
			{
			ReadOptionL(aLex, aFileName);
			}
		else if (command == KCmndInclude)
			{
			if (iParent == NULL)
				{
				iProcessInclude = EFalse;
				TLex lineLex(LineRemainder(aLex));
				TPtrC fileName(NextWord(lineLex));
				TFileName2* fullFileName = new(ELeave) TFileName2(aFileName);
				CleanupStack::PushL(fullFileName);
				fullFileName->SetNameAndExtL(fileName);
				ReadFileL(aFs, *fullFileName);
				CleanupStack::PopAndDestroy(fullFileName);
				break;
				}
			else
				{
				// We're a sub-command. Let control return to the root to handle the include.
				aLex.UnGetToMark(mark);
				iParent->ProcessInclude(*this);
				break;
				}
			}
		else if (command == KCmndSubCommand)
			{
			if (iParent == NULL)
				{
				TLex lineLex(LineRemainder(aLex));
				AddSubCommandL(lineLex, aLex, aFs, aFileName);
				}
			else
				{
				// We're a sub-command. Let control return to the root to handle the next sub-command.
				aLex.UnGetToMark(mark);
				iParent->ProcessNewChild();
				break;
				}
			}
		else
			{
			StaticLeaveIfErr(KErrArgument, _L("Unknown command \"%S\" in \"%S\""), &command, &aFileName);
//.........这里部分代码省略.........

/**
Search fixed start URLs, i.e. URLs without schema (www., wap.).
Also finds IPv4 addresses (*.*.*.*).
As a special case, supports deprecated hardcoded schematic addresses finding 
(http://, https://, rtsp://) to make sure deprecated search cases work 
as they did previously.

@param aText Text that will be parsed
@param aFindFixedSchemas If true, will find old fixed schematic URLs also
@return ETrue if any URL are found else returns EFalse
@leave KErrNone, if successful; otherwise one of the other system-wide error codes.
@panic ETulPanicDescriptorLength in debug build if item's position and/or length is out of the document's range.
*/
TBool CTulAddressStringTokenizer::SearchUrlL( const TDesC& aText, const TBool aFindFixedSchemas )
    {
    TLex text = aText;
    while ( !text.Eos() )
        {
        while( !(text.Eos()) && !IsValidUrlChar( text.Peek() ) )
            text.Inc();

        text.Mark();
        while( !(text.Eos()) && IsValidUrlChar( text.Peek() ) )
            text.Inc();

        TPtrC tokenPtr = text.MarkedToken();
        TBool wasValidUrl = EFalse;

        if ( aFindFixedSchemas )	// Search for http://
            wasValidUrl = ParseUrlL( KHttpUrlAddress, tokenPtr, text.Offset() );
        
        if (aFindFixedSchemas && !wasValidUrl)	// Search for https://
            wasValidUrl = ParseUrlL( KHttpsUrlAddress, tokenPtr, text.Offset() );

        if (aFindFixedSchemas && !wasValidUrl) // Search for rtsp://
            wasValidUrl = ParseUrlL( KRtspUrlAddress, tokenPtr, text.Offset() );

        if ( !wasValidUrl )	// Search for www.
            wasValidUrl = ParseUrlL( KWwwUrlAddress, tokenPtr, text.Offset() );

        if ( !wasValidUrl )	// Search for wap.
            wasValidUrl = ParseUrlL( KWapUrlAddress, tokenPtr, text.Offset() );

        if ( !wasValidUrl )	// Search for IP-address (xxx.xxx.xxx.xxx)
            { 
            if ( tokenPtr.Match( KIPAddress ) != KErrNotFound )
                {
                TInt periods = 0;
                wasValidUrl = ETrue;
                TBool endWithPunctuation = EFalse;
                TBool betweenBrackets = EFalse;

                // First see if token ends with ",",".","!","?",";" or ":"
                TChar charac = tokenPtr[tokenPtr.Length() - 1];
                TChar charac0 = tokenPtr[0];
                if ( charac == ',' || charac == '.' ||
                     charac == '!' || charac == '?' ||
                     charac == ';' || charac == ':' )
                    {
                    endWithPunctuation = ETrue;
                    }
                // Or if it starts and ends with brackets or quotation marks
                else if ( ( charac0 == '(' && charac == ')' )
                       || ( charac0 == '"' && charac == '"' )
                       || ( charac0 == '[' && charac == ']' )
                       || ( charac0 == '<' && charac == '>' ) )
                    {
                    betweenBrackets = ETrue;
                    }

                TInt i = 0;
                TInt tokensEnd = tokenPtr.Length();
                if ( endWithPunctuation )
                    tokensEnd--;
                else if ( betweenBrackets )
                    {
                    i = 1;
                    tokensEnd--;
                    }

                // Take a closer look to see if a valid IP-address
                TBuf<3> ipPart;
                TInt numbers = 0;
                for ( ; i < tokensEnd; i++ )
                    {
                    if ( !( ((TChar)tokenPtr[i]).IsDigit() || tokenPtr[i] == '.' ) )
                        {
                        wasValidUrl = EFalse;
                        break;
                        }

                    if ( tokenPtr[i] == '.' )
                        periods++;
                    else
                        numbers++;

                    if ( numbers > KNumbersInIpAddress || periods > KDotsInIpAddress )
                        {
                        wasValidUrl = EFalse;
                        break;
//.........这里部分代码省略.........

//.........这里部分代码省略.........
        number.Inc();
        // ------------------------------------
        number.Mark(endMark);

        // If they exist, remove brackets from the beginning and the end
        number.Mark(mark); // Let's mark where to continue the search
        TBool endBrackets = ETrue;
        do
            {
            number.UnGet();

            if ( number.Peek() == ')' )
                {
                number.UnGetToMark(startMark);
                if ( number.Peek() == '(' )
                    {
                    // If there's more than one pair of brackets -> don't strip them.
                    if ( brackets > 1 )
                        break;

                    number.Inc();
                    number.Mark(startMark);
                    number.UnGetToMark(endMark);
                    number.UnGet();
                    number.Mark(endMark);
                    // Get rid of whitespaces and periods from the end and from the beginning
					number.UnGet();
                    while ( (number.Peek().IsSpace() || number.Peek() == '.') 
                            && number.Offset() > number.MarkedOffset(startMark) )     
                        { // from the end
                        number.UnGet();
                        }
					number.Inc();
                    number.Mark(endMark);
                    number.UnGetToMark(startMark);
                    while ( (number.Peek().IsSpace() || number.Peek() == '.') 
                            && number.Offset() < number.MarkedOffset(endMark) )     
                        { // from the beginning
                        number.Inc();
                        }
                    number.Mark(startMark);
                    number.UnGetToMark(endMark);
                    // ----
                    }
                else
                    endBrackets = EFalse;
                }
            else
                endBrackets = EFalse;
            }
        while ( endBrackets );

        number.UnGetToMark(mark);
        // ----------------        

        if ( numberCount <= KFindItemMaxNumbers && numberCount >= iMinNumbers )
            {
			TPtrC tokenPtr = number.MarkedToken(startMark);
			TInt tokensEnd = tokenPtr.Length();
			TInt numbers = 0;
			TInt partialNumber = 0;
			TBool wasValidPhoneNumber = ETrue;
			TInt i = 0;

			for ( ; i < tokensEnd; i++ )
				{
				if ( tokenPtr[i] == '.' )
					partialNumber = 0;
				else if ( ((TChar)tokenPtr[i]).IsDigit() )
					{
					numbers++;
					partialNumber++;
					}

				if ( ( partialNumber == 1 || partialNumber == 2 ) && i + 1 < tokensEnd )
					{
					if ( tokenPtr[i + 1] == '.' )
						wasValidPhoneNumber = EFalse;
					}
				}
				
			if (!wasValidPhoneNumber && numbers > 6)
				wasValidPhoneNumber = ETrue;
			
			if (wasValidPhoneNumber)
				{
	            __ASSERT_DEBUG( number.MarkedOffset(startMark) + number.MarkedOffset(endMark) 
	                            - number.MarkedOffset(startMark) <= aText.Length(), 
	                            Panic(ETulPanicDescriptorLength) );

	            AddItemL( number.MarkedOffset(startMark), 
	                      number.MarkedOffset(endMark) - number.MarkedOffset(startMark), 
	                      EFindItemSearchPhoneNumberBin );
				}
			}

        }

    return (iFoundItems->Count() > 0);
    }

/**
Search algorithm for searching generic URIs

@param aText Text that will be parsed
@return ETrue if any generic URI is found else returns EFalse
@leave KErrNone, if successful; otherwise one of the other system-wide error codes.
*/
TBool CTulAddressStringTokenizer::SearchGenericUriL( const TDesC& aText )
    {
    // Detect generic URI within the token
    const TDesC& schemeStartArray = KURISchemeStartCharacters;
    const TDesC& schemeBodyArray = KURISchemeBodyCharacters;
    const TDesC& schemeTerminatorArray = KURISchemeTerminator;
    const TDesC& URIArray = KURICharacters;

    TBool wasValidUri = EFalse;
    TLex text = aText;

    while ( !text.Eos() )
        {
        // Discard characters until URI scheme terminator is found
        while( !(text.Eos()) && schemeTerminatorArray.Locate(text.Peek()) == KErrNotFound )
            text.Inc();

        // if at end of the text, no legit URI found
        if ( !text.Eos() )
            {
            // Store the schema end offset (+1 to skip ':')
            TInt schemeEndOffset = text.Offset() + 1;

            // Scheme must be at least 1 character long at the beginning of the text to be valid
            if ( text.Offset() > 0 ) 
                {
                // Un-get last scheme character to begin examination
                text.UnGet();

                // Rewind until beginning of the URI
                while ( text.Offset() > 0 && schemeBodyArray.Locate(text.Peek().GetLowerCase()) != KErrNotFound )
                    text.UnGet();
        
                // Now text pointer is at first character of the URI
                // Do go back through the scheme until a legal beginning character for URI 
                // is found or back to the (schemeEndOffset - 1) i.e. URI scheme terminator
                while ( schemeStartArray.Locate(text.Peek().GetLowerCase()) == KErrNotFound && (text.Offset() + 1) < schemeEndOffset )
                    text.Inc();

                // check if terminated because a valid start character was found when
                // scheme terminator was reached.
                if ( schemeStartArray.Locate(text.Peek().GetLowerCase()) != KErrNotFound )
                    {
                    // First character is a valid URI char, so the scheme is valid -> 
                    // marks the beginning of the array
                    text.Mark();
            
                    // fast forward to the end of the scheme
                    while( text.Offset() < schemeEndOffset )    
                        text.Inc();
            
                    // Get characters until end of schema
                    while( !(text.Eos()) && URIArray.Locate( text.Peek().GetLowerCase() ) != KErrNotFound )
                        text.Inc();

                    // remove certain punctuation from end of the URI, as it is likely 
                    // to be part of the surrounding text.
                    text.UnGet();

                    //special processing for bracket
                    //only remove the end bracket if there is no open bracket in the uri
                    //not counting bracket pairs for efficiency
                    if (text.Peek()!=')' || text.MarkedToken().Locate(TChar('('))!=-1)
                        text.Inc();

                    text.UnGet();
                    if ( text.Peek() != '.' && text.Peek() != '?' && text.Peek() != ',')
                        text.Inc();
                
                    // URI cannot contain only scheme, so check that pointer was increased 
                    // by at least one character
                    if ( schemeEndOffset != text.Offset() )
                        {
                        // Append found text to item array (it is now known to be 
                        // syntactically valid URI as it contains characters after the scheme)
                        AddItemL( text.MarkedOffset(), text.Offset() - text.MarkedOffset(), EFindItemSearchScheme );
                        wasValidUri = ETrue;
                        }
                    }
                else // First character of scheme is not legit, fast forward to end of the 
                     // scheme anyway to continue search
                    {
                    while( text.Offset() < schemeEndOffset )        
                        text.Inc();
                    }
                }
            else
 				text.Inc();
            }
        }

    return wasValidUri;
    }