C++ ParseToken函数代码示例

void RawParser::ParseInfoToken(const char *&pcs, int &iLen, bool *pbMoreFollows)
{
	if (!FindNextToken()) {
		if (pbMoreFollows) *pbMoreFollows = false;
		return;
	}

	pcs = (const char *)pbPos;
	if (*pcs == '"') pcs++;

	ParseToken(sz, pbMoreFollows);

	iLen = int(strlen(sz));
}

// Initialize net service list
void InitNetSvcList(CEDAR *cedar)
{
	char filename[MAX_PATH] = "/etc/services";
	BUF *b;
	// Validate arguments
	if (cedar == NULL)
	{
		return;
	}

#ifdef	OS_WIN32
	Format(filename, sizeof(filename), "%s\\drivers\\etc\\services", MsGetSystem32Dir());
#endif

	cedar->NetSvcList = NewList(CompareNetSvc);

	b = ReadDump(filename);
	if (b == NULL)
	{
		return;
	}

	while (true)
	{
		char *s = CfgReadNextLine(b);
		if (s == NULL)
		{
			break;
		}

		Trim(s);
		if (s[0] != '#')
		{
			TOKEN_LIST *t = ParseToken(s, " \t/");
			if (t->NumTokens >= 3)
			{
				NETSVC *n = ZeroMalloc(sizeof(NETSVC));
				n->Name = CopyStr(t->Token[0]);
				n->Udp = (StrCmpi(t->Token[2], "udp") == 0 ? true : false);
				n->Port = ToInt(t->Token[1]);
				Add(cedar->NetSvcList, n);
			}
			FreeToken(t);
		}
		Free(s);
	}

	FreeBuf(b);
}

int Anasin::ListidP(){
	Lexema nextLexema = this->_symTab->lookAheadLexema();
	if(nextLexema._tipo == "Punctuation" && nextLexema._valor == ","){
		//12) ListidP -> , Listid
		if(!ParseToken(Lexema("Punctuation",","),true)) return 0;
		if(!Listid()) return 0;
	}else if(nextLexema._tipo == "Reserved" && nextLexema._valor == "nil"){
		//13) ListidP -> €
		// No hago nada
	}else{
		std::cout << "Error sintáctico, se esperaba : Punctuation, nil y se obtuvo: " << nextLexema._tipo << " ("<< nextLexema._valor <<")["<< this->_symTab->getOffset() <<"]" << std::endl;
		return 0;
	}
	return 1;
}

Value* JSONReader::JsonToValue(const std::string& json, bool check_root,
                               bool allow_trailing_comma)
{
    // 输入必须是UTF-8编码.
    if(!IsStringUTF8(json.c_str()))
    {
        error_code_ = JSON_UNSUPPORTED_ENCODING;
        return NULL;
    }

    // 从UTF8到wstring的转换会移除空字节(好事).
    std::wstring json_wide(UTF8ToWide(json));
    start_pos_ = json_wide.c_str();

    // 当输入的JSON字符串开头有UTF-8的Byte-Order-Mark(0xEF, 0xBB, 0xBF),
    // UTF8ToWide()函数会把它转换成BOM(U+FEFF). 为防止JSONReader::BuildValue()
    // 函数把它当成非法字符而返回NULL, 如果存在Unicode的BOM则先跳过.
    if(!json_wide.empty() && start_pos_[0]==0xFEFF)
    {
        ++start_pos_;
    }

    json_pos_ = start_pos_;
    allow_trailing_comma_ = allow_trailing_comma;
    stack_depth_ = 0;
    error_code_ = JSON_NO_ERROR;

    scoped_ptr<Value> root(BuildValue(check_root));
    if(root.get())
    {
        if(ParseToken().type == Token::END_OF_INPUT)
        {
            return root.release();
        }
        else
        {
            SetErrorCode(JSON_UNEXPECTED_DATA_AFTER_ROOT, json_pos_);
        }
    }

    // "语法错误".
    if(error_code_ == 0)
    {
        SetErrorCode(JSON_SYNTAX_ERROR, json_pos_);
    }

    return NULL;
}

eTokenType asCTokenizer::GetToken(const char *source, size_t sourceLength, size_t *tokenLength, asETokenClass *tc)
{
	asASSERT(source != 0);
	asASSERT(tokenLength != 0);

	this->source = source;
	this->sourceLength = sourceLength;

	asETokenClass t = ParseToken();
	if( tc ) *tc = t;

	// Copy the output to the token
	*tokenLength = this->tokenLength;

	return tokenType;
}

static int
ParseExtendedTX(void)
{
    int style = TX_ORIGINAL;

    if (ParseToken(PARSE_COMMENT)) {
	if (!strncmp(token, "//TX", 4)) {
	    if (token[4] == '1')
		style = TX_QUARK_TYPE1;
	    else if (token[4] == '2')
		style = TX_QUARK_TYPE2;
	}
    }

    return style;
}

static texcoord_style_t
ParseExtendedTX(parser_t *parser)
{
    texcoord_style_t style = TX_ORIGINAL;

    if (ParseToken(parser, PARSE_COMMENT)) {
	if (!strncmp(parser->token, "//TX", 4)) {
	    if (parser->token[4] == '1')
		style = TX_QUARK_TYPE1;
	    else if (parser->token[4] == '2')
		style = TX_QUARK_TYPE2;
	}
    }

    return style;
}

void CommandLineMode(int argc, char *argv[], void *htmsocket) {
    char *messageName;
    char *token;
    typedArg args[MAX_ARGS];
    int i,j, numArgs;
    OSCbuf buf[1];

    OSC_initBuffer(buf, SC_BUFFER_SIZE, bufferForOSCbuf);

    if (argc > 1) {
	if (OSC_openBundle(buf, OSCTT_Immediately())) {
	    complain("Problem opening bundle: %s\n", OSC_errorMessage);
	    return;
	}
    }

    for (i = 0; i < argc; i++) {
        messageName = strtok(argv[i], ",");
        if (messageName == NULL) {
            break;
        }

        j = 0;
        while ((token = strtok(NULL, ",")) != NULL) {
            args[j] = ParseToken(token);
            j++;
	    if (j >= MAX_ARGS) {
		complain("Sorry; your message has more than MAX_ARGS (%d) arguments; ignoring the rest.\n",
			 MAX_ARGS);
		break;
	    }
        }
        numArgs = j;

        WriteMessage(buf, messageName, numArgs, args);
    }

    if (argc > 1) {
	if (OSC_closeBundle(buf)) {
	    complain("Problem closing bundle: %s\n", OSC_errorMessage);
	    return;
	}
    }

    SendBuffer(htmsocket, buf);
}

int Anasin::Operador(){
	Lexema nextLexema = this->_symTab->lookAheadLexema();
	if(nextLexema._tipo == "Arithmetic"){
		// 27) Operador  -> arithmetic Expresion
		if(!ParseToken(Lexema("Arithmetic",""),false)) return 0;
		if(!Expresion()) return 0;
	}else if(nextLexema._tipo == "Relational" ||
			(nextLexema._tipo == "Agrupation" && nextLexema._valor == ")") ||
			(nextLexema._tipo == "Punctuation" && nextLexema._valor == ";")){
		// 28) Operador  -> €
		// No hago nada
	}else{
		std::cout << "Error sintáctico, se esperaba : Arithmetic, Relational, ) o ; y se obtuvo: " << nextLexema._tipo << " ("<< nextLexema._valor <<")["<< this->_symTab->getOffset() <<"]" << std::endl;
		return 0;
	}
	return 1;
}

int DoInclude(ParsePtr p)
{
    DynamicBuffer buf;
    int r, e;

    DBufInit(&buf);
    if ( (r=ParseToken(p, &buf)) ) return r;
    e = VerifyEoln(p);
    if (e) Eprint("%s", ErrMsg[e]);
    if ( (r=IncludeFile(DBufValue(&buf))) ) {
	DBufFree(&buf);
	return r;
    }
    DBufFree(&buf);
    NumIfs = 0;
    IfFlags = 0;
    return OK;
}

bool CXMLElement::ParseRootTag (IMemoryBlock &Stream, CString *retsTag)

//	ParseRootTag
//
//	This function parses only enough to determine the root tag, either by reading
//	as far as the first open tag or by getting the DOCTYPE.
//
//	This function is a hack to allow Transcendence to read the root tag for an
//	extension without loading the whole file.

	{
	//	Initialize context

	ParserCtx Ctx(Stream, NULL);
	Ctx.m_bParseRootTag = true;

	//	Parse the prologue

	if (!ParsePrologue(&Ctx))
		return false;

	if (!Ctx.m_sRootTag.IsEmpty())
		{
		*retsTag = Ctx.m_sRootTag;
		return true;
		}

	//	Next token must be an element open tag

	if (Ctx.iToken != tkTagOpen)
		return false;

	//	Parse the root element name

	if (ParseToken(&Ctx) != tkText)
		return false;

	*retsTag = Ctx.sToken;

	//	Done

	return true;
	}

int CCloseCaptionPhrase::CountTokens( void ) const
{
	int count = 0;
	wchar_t token[ 1024 ];
	wchar_t const *in = GetStream();

	while ( in )
	{
		in = ParseToken( in, token, sizeof( token ) );
		if ( wcslen( token ) > 0 )
		{
			count++;
		}
		if ( !in || *in == L'\0' )
			break;
	}

	return count;
}

bool Scanner::GetToken(Token *token)
{
	if ((token == NULL))
	{
		return false;
	}

	Skip();

	if (ParseDigit(token))
		return true;

	if (ParseToken(token))
		return true;

	if (ParseOperator(token))
		return true;

	return false;
}

bool RawParser::ParseParams(PCTCHAR pcszDefaultFile)
{
	if (!ParseToken(sz)) return false;
	if (_strcmpi(sz, "RawImage") != 0) return false;

	for (int n = 0; n < 4; n++) {
		_tcscpy(Comps[n].szFile, pcszDefaultFile);
		Comps[n].nStart = 0;
		Comps[n].nDelta = -1;
	}

	ParseMainBlock();

	for (int n = 0; n < 4; n++) {
		if (Comps[n].nDelta < 0)
			Comps[n].nDelta = cbComp;
	}

	return true;
}

static inline
int ParseFile(char* Filename, GLArbToken* ArbHash,
               GLToken* FunctionsHash, unsigned int* FunctionCount,
               GLToken* DefinesHash, unsigned int* DefinesCount,
               GLSettings* Settings)
{
  char* Data = ReadEntireFile(Filename);
  int Success = 0;
  if (Data)
  {
    GLTokenizer Tokenizer;
    Tokenizer.At = Data;
    while(*Tokenizer.At)
    {
      GLToken Token = ParseToken(&Tokenizer);
      if (StartsWith(Token.Value, "gl") && IsUpperCase(Token.Value.Chars[2]))
      {
        if (!Contains(FunctionsHash, Token) && IsKnownOrIgnoredToken(ArbHash, &Token, Settings))
        {
          AddToken(FunctionsHash, Token);
          *FunctionCount += 1;
        }
      }
      if (StartsWith(Token.Value, "GL_"))
      {
        if (!Contains(DefinesHash, Token) && IsKnownOrIgnoredToken(ArbHash, &Token, Settings))
        {
          AddToken(DefinesHash, Token);
          *DefinesCount += 1;
        }
      }
    }
    free(Data);
    Success = 1;
  }
  else
  {
    fprintf(stderr, "Couldn't open file %s", Filename);
  }
  return Success;
}