C++ FindToken函数代码示例

bool TokenName( unsigned token, char **start, unsigned *len )
{
    switch( token ) {
    case T_LINE_SEPARATOR:
        *start = LIT( End_Of_Line );
        *len = strlen( LIT( End_Of_Line ) ) + 1;
        return( TRUE );
    case T_INT_NUM:
    case T_REAL_NUM:
        *start = LIT( Num_Name );
        *len = strlen( LIT( Num_Name ) ) + 1;
        return( TRUE );
    case T_NAME:
        *start = LIT( Sym_Name_Name );
        *len = strlen( LIT( Sym_Name_Name ) ) + 1;
        return( TRUE );
    }
    if( token < LAST_CMDLN_DELIM ) {
        *start = &CmdLnDelimTab[ token - FIRST_CMDLN_DELIM ];
        *len = sizeof( char );
        return( TRUE );
    }
    if( ExprTokens != NULL ) {
        if( FindToken( ExprTokens->delims, token, start, len ) ) return(TRUE);
        if( FindToken( ExprTokens->keywords, token, start, len ) ) return(TRUE);
    }
    return( FALSE );
}

void Parser::SearchForEpisodeTitle() {
  auto token_begin = tokens_.begin();
  auto token_end = tokens_.begin();

  do {
    // Find the first non-enclosed unknown token
    token_begin = FindToken(token_end, tokens_.end(),
                            kFlagNotEnclosed | kFlagUnknown);
    if (token_begin == tokens_.end())
      return;

    // Continue until a bracket or identifier is found
    token_end = FindToken(token_begin, tokens_.end(),
                          kFlagBracket | kFlagIdentifier);

    // Ignore if it's only a dash
    if (std::distance(token_begin, token_end) <= 2 &&
        IsDashCharacter(token_begin->content)) {
      continue;
    }

    // Build episode title
    BuildElement(kElementEpisodeTitle, false, token_begin, token_end);
    return;
  } while (token_begin != tokens_.end());
}

void Parser::SearchForReleaseGroup() {
  token_container_t::iterator token_begin = tokens_.begin();
  token_container_t::iterator token_end = tokens_.begin();

  do {
    // Find the first enclosed unknown token
    token_begin = FindToken(token_end, tokens_.end(),
                            kFlagEnclosed | kFlagUnknown);
    if (token_begin == tokens_.end())
      return;

    // Continue until a bracket or identifier is found
    token_end = FindToken(token_begin, tokens_.end(),
                          kFlagBracket | kFlagIdentifier);
    if (token_end->category != kBracket)
      continue;

    // Ignore if it's not the first non-delimiter token in group
    token_container_t::iterator previous_token = FindPreviousToken(tokens_, token_begin,
                                            kFlagNotDelimiter);
    if (previous_token != tokens_.end() &&
        previous_token->category != kBracket) {
      continue;
    }

    // Build release group
    BuildElement(kElementReleaseGroup, true, token_begin, token_end);
    return;
  } while (token_begin != tokens_.end());
}

void TLexer::Read() {
    char c;
    std::string value = "";
    while (true) {
        c = GetChar();
        if (FileEOF()) {
            break;
        }
        value += c;
        switch (c) {
            case 34: {
                ReadString(value);
                break;
            }

            case 39: {
                ReadSymbol(value);
                break;
            }

            case '.' : {
                FindToken(State_Point, value);
                break;
            }

            case '(': {
                FindToken(State_Lbkt,value);
                break;
            }

            case ')': {
                FindToken(State_Rbkt,value);
                break;
            }
            case '#': {
                ReadSharp(value);
                break;
            }
            default: {
                if (isspecial(c) || isalpha(c)) {
                    ReadIdent(value);
                } else if (isdigit(c)) {
                    ReadNumber(value);
                } else if (IsEndToken(c)) {
                    value.erase(value.size() - 1, value.size());
                }
                break;
            }
        }
    }
    value = "EOF";
    FindToken(State_EOF, value);
}

void MenuManager::ParsePageInfo(char* plinePtr, int* menuNumber, int* pageNumber, int* itemNumber)
{	
	(*pageNumber)++;
	*itemNumber = -1;

	char	pdrawColor[20];
	ULONG color;
	char	pseparators[] = {0x20, 0x2c, 0x3d, 0x3b, 0x0d, 0x0a, 0x09, 0x23, 0x00};
	char	*ptoken;

	PageStruct* pPage = &(mpMenus[*menuNumber].mpPages[*pageNumber]);

	#ifdef USE_SH_POOLS
	pPage->mpTitle = (char *)MemAllocPtr(gCockMemPool,sizeof(char)*mMaxTextLen,FALSE);
	#else
	pPage->mpTitle		= new char[mMaxTextLen];
	#endif

	*(pPage->mpTitle)	= '\0';

	ptoken = FindToken(&plinePtr, pseparators);	
	sscanf(ptoken, "%d",  &(pPage->mNumItems));
	ptoken = FindToken(&plinePtr, pseparators);	
	sscanf(ptoken, "%d",  &(pPage->mCondition));
	ptoken = FindToken(&plinePtr, pseparators);	
	sscanf(ptoken, "%s",  pdrawColor);

	*(pPage->mpTitle) = NULL;
	ptoken = FindToken(&plinePtr, pseparators);	
	while(ptoken) {
		strcat(pPage->mpTitle, ptoken);
		ptoken = FindToken(&plinePtr, pseparators);
		if(ptoken) {
			strcat(pPage->mpTitle, " ");
		}
	}

	if(FindMenuColorValue(pdrawColor, &color)) {
		pPage->mDrawColor = color;
	}
	else {
		pPage->mDrawColor = gMenuColorTable[MENU_WHITE].mValue;
	}


	#ifdef USE_SH_POOLS
	pPage->mpItems = (ItemStruct *)MemAllocPtr(gCockMemPool,sizeof(ItemStruct)*pPage->mNumItems,FALSE);
	#else
	pPage->mpItems = new ItemStruct[pPage->mNumItems];
	#endif
}

L_TOKEN* IDENTIFIER(char*          source,
                    int            start,
                    int            end,
                    int            line,
                    GRAMMAR_TABLE  grammar)
{
    int token;
    L_TOKEN* block = (L_TOKEN*)malloc(sizeof(L_TOKEN));

    token = FindToken(&source[start], end - start, grammar);
    if (token) {
        // token
        block->token = token;
        block->string = source + start;
        block->length = end - start;
        block->line = line;
        block->next = 0;
    } else {
        // identifier
        block->token = gSymbolIdentifier;
        block->string = source + start;
        block->length = end - start;
        block->line = line;
        block->next = 0;
    }

    return block;
}

L_TOKEN* TOKEN(char*          source,
               int            start,
               int            end,
               int            line,
               GRAMMAR_TABLE  grammar)
{
    int token;
    L_TOKEN* block = (L_TOKEN*)malloc(sizeof(L_TOKEN));

    token = FindToken(&source[start], end - start, grammar);
    if (token) {
        // token
        block->token = token;
        block->string = source + start;
        block->length = end - start;
        block->line = line;
        block->next = 0;
    } else {
        // identifier
        free(block);
        block = NULL;
        printf("Syntax error on line %i, ", line+1);
        printf("Error, illegal identifier: ");
        while (start < end)
            printf("0x%X", source[start++]);
        printf("\n");
    }

    return block;
}

token_iterator_t FindPreviousToken(token_container_t& tokens,
                                   token_iterator_t first,
                                   unsigned int flags) {
  auto it = FindToken(std::reverse_iterator<token_iterator_t>(first),
                       tokens.rend(), flags);
  return it == tokens.rend() ? tokens.end() : (++it).base();
}

int StyleSheetParser::Parse(StyleSheetNode* node, Stream* _stream)
{
	int rule_count = 0;
	line_number = 0;
	stream = _stream;
	stream_file_name = stream->GetSourceURL().GetURL().Replace("|", ":");

	// Look for more styles while data is available
	while (FillBuffer())
	{
		String style_names;
		
		while (FindToken(style_names, "{", true))
		{
			// Read the attributes
			PropertyDictionary properties;
			if (!ReadProperties(properties))
			{
				continue;
			}

			StringList style_name_list;
			StringUtilities::ExpandString(style_name_list, style_names);

			// Add style nodes to the root of the tree
			for (size_t i = 0; i < style_name_list.size(); i++)
				ImportProperties(node, style_name_list[i], properties, rule_count);

			rule_count++;
		}
	}	

	return rule_count;
}

void pgSettingItem::SetType(const wxString &str)
{
	int index = FindToken(str, pgConfigTypeStrings);
	if (index < 0)
		type = PGC_STRING;
	else
		type = (pgConfigType)index;
}

void TLexer::IsCorrectToken(char c, std::string& value, States state) {
    if (IsEndToken(c)) {
        FindToken(state, value);
    } else {
        value += c;
        throw new ExceptionLexerIncorrectChar(value, PosLine, PosColumn);
    }
}

struct TokenStruct NextToken (char *c) {
	*c = FindToken (*c);
	struct TokenStruct ret;// = malloc (sizeof (struct TokenStruct));
	ret.Token = malloc (sizeof (char) * 2);
	ret.Token[0] = *c;
	ret.type = TokenType (*c);
	ret.priority = OpPriority (*c);
	ret.index = line_index;
	
	int length = 1;
	if (!ret.type) {
		if (*c == '+' || *c == '-' || *c == '*' || *c == '/' || *c == '%' || *c == '^' || *c == '=' || *c == ')' || *c == '(' || *c == ',') {	// Grab next character if we have a valid token
			*c = FindToken (0);
		}
		ret.Token[1] = '\0';
#ifdef DEBUG
		if (ret.Token[0] != '\n') { printf ("	Op/Invalid Token %i: '%s';\n", ret.index, ret.Token); }
		else { printf ("	Op/Invalid Token %i: '\\n';\n", ret.index); }
		sleep (1);
#endif
		return ret;
	}
	
	*c = getchar ();
	AddSpace (*c);
	while (TokenType (*c) == ret.type) {
		ret.Token[length++] = *c;
		if (ECHO) { putchar (*c); }
		ret.Token = realloc (ret.Token, sizeof (char) * (length+1));
		*c = getchar ();
		AddSpace (*c);
	}
	if (ECHO) { putchar (*c); }
	//line_index += length;
	ret.Token[length] = '\0';
	*c = FindToken (*c);
#ifdef DEBUG
	printf ("	Var/Num Token %i: %s;\n", ret.index, ret.Token);
	sleep (1);
#endif
	return ret;
}

void MenuManager::ReadDataFile(char* pfileName)
{
	FILE*				pFile;
	BOOL				quitFlag  = FALSE;
	char*				presult = "";
	char*				plinePtr;
	const int		lineLen = MAX_LINE_BUFFER - 1;
	char				plineBuffer[MAX_LINE_BUFFER] = "";
	char				*ptoken;
	char				pseparators[] = {0x20, 0x2c, 0x3d, 0x3b, 0x0d, 0x0a, 0x09, 0x23, 0x00};
	int				menuNumber = -1;
	int				pageNumber = -1;
	int				itemNumber = -1;

	pFile = CP_OPEN(pfileName, "r");
	F4Assert(pFile);

	presult	= fgets(plineBuffer, lineLen, pFile);
	quitFlag	= (presult == NULL);
	plinePtr = plineBuffer;

	while(!quitFlag) {

		if(*plineBuffer == '#') {

			ptoken = FindToken(&plinePtr, pseparators);	

			if(!strcmpi(ptoken, TYPE_MENUMANGER_STR)) {
				ParseManagerInfo(plinePtr);
			}
			else if(!strcmpi(ptoken, TYPE_MENU_STR)) {
				ParseMenuInfo(plinePtr, &menuNumber, &pageNumber);
			}
			else if(!strcmpi(ptoken, TYPE_PAGE_STR)) {
				ParsePageInfo(plinePtr, &menuNumber, &pageNumber, &itemNumber);
			}
			else if(!strcmpi(ptoken, TYPE_ITEM_STR)) {
				ParseItemInfo(plinePtr, &menuNumber, &pageNumber, &itemNumber);
			}
			else if(!strcmpi(ptoken, TYPE_RES_STR)) {
				ParseResInfo(plinePtr, --mResCount);
			}
			else {
				ShiWarning ("Bad String in menu file");
			}
		}
		presult	= fgets(plineBuffer, lineLen, pFile);
		plinePtr = plineBuffer;
		quitFlag	= (presult == NULL);
	}

	CP_CLOSE(pFile);
	ShiAssert(mResCount == 0);	// Bad data in the manager information line!
}

void PdfParser::ReadXRef( pdf_long* pXRefOffset )
{
    FindToken( "startxref", PDF_XREF_BUF );

    if( !this->IsNextToken( "startxref" ) )
    {
        PODOFO_RAISE_ERROR( ePdfError_NoXRef );
    }

    *pXRefOffset = this->GetNextNumber();
}

void ExplicitConstructorCheck::check(const MatchFinder::MatchResult &Result) {
  const CXXConstructorDecl *Ctor =
      Result.Nodes.getNodeAs<CXXConstructorDecl>("ctor");
  // Do not be confused: isExplicit means 'explicit' keyword is present,
  // isImplicit means that it's a compiler-generated constructor.
  if (Ctor->isOutOfLine() || Ctor->isImplicit() || Ctor->isDeleted() ||
      Ctor->getNumParams() == 0 || Ctor->getMinRequiredArguments() > 1)
    return;

  bool takesInitializerList = isStdInitializerList(
      Ctor->getParamDecl(0)->getType().getNonReferenceType());
  if (Ctor->isExplicit() &&
      (Ctor->isCopyOrMoveConstructor() || takesInitializerList)) {
    auto isKWExplicit = [](const Token &Tok) {
      return Tok.is(tok::raw_identifier) &&
             Tok.getRawIdentifier() == "explicit";
    };
    SourceRange ExplicitTokenRange =
        FindToken(*Result.SourceManager, Result.Context->getLangOpts(),
                  Ctor->getOuterLocStart(), Ctor->getLocEnd(), isKWExplicit);
    StringRef ConstructorDescription;
    if (Ctor->isMoveConstructor())
      ConstructorDescription = "move";
    else if (Ctor->isCopyConstructor())
      ConstructorDescription = "copy";
    else
      ConstructorDescription = "initializer-list";

    DiagnosticBuilder Diag =
        diag(Ctor->getLocation(),
             "%0 constructor should not be declared explicit")
        << ConstructorDescription;
    if (ExplicitTokenRange.isValid()) {
      Diag << FixItHint::CreateRemoval(
          CharSourceRange::getCharRange(ExplicitTokenRange));
    }
    return;
  }

  if (Ctor->isExplicit() || Ctor->isCopyOrMoveConstructor() ||
      takesInitializerList)
    return;

  bool SingleArgument =
      Ctor->getNumParams() == 1 && !Ctor->getParamDecl(0)->isParameterPack();
  SourceLocation Loc = Ctor->getLocation();
  diag(Loc,
       "%0 must be marked explicit to avoid unintentional implicit conversions")
      << (SingleArgument
              ? "single-argument constructors"
              : "constructors that are callable with a single argument")
      << FixItHint::CreateInsertion(Loc, "explicit ");
}