本文整理汇总了C++中CLanguage::Token方法的典型用法代码示例。如果您正苦于以下问题：C++ CLanguage::Token方法的具体用法？C++ CLanguage::Token怎么用？C++ CLanguage::Token使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类CLanguage的用法示例。
在下文中一共展示了CLanguage::Token方法的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C++代码示例。
示例1: ParseException
Token&
Tokenizer::_ParseHexOperand()
{
const char* begin = fCurrentChar;
fCurrentChar += 2;
// skip "0x"
if (!_IsHexDigit(*fCurrentChar))
throw ParseException("expected hex digit", _CurrentPos());
fCurrentChar++;
while (_IsHexDigit(*fCurrentChar))
fCurrentChar++;
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
if (length <= 10) {
// including the leading 0x, a 32-bit constant will be at most
// 10 characters. Anything larger, and 64 is necessary.
fCurrentToken.value.SetTo((uint32)strtoul(
fCurrentToken.string.String(), NULL, 16));
} else {
fCurrentToken.value.SetTo((uint64)strtoull(
fCurrentToken.string.String(), NULL, 16));
}
return fCurrentToken;
}示例2: Token
void
Tokenizer::SetTo(const char* string)
{
fString = string;
fCurrentChar = fString.String();
fCurrentToken = Token();
fReuseToken = false;
}示例3: _ParseHexOperand
const Token&
Tokenizer::NextToken()
{
if (fCurrentToken.type == TOKEN_END_OF_LINE)
return fCurrentToken;
if (fReuseToken) {
fReuseToken = false;
return fCurrentToken;
}
while (*fCurrentChar != 0 && isspace(*fCurrentChar))
fCurrentChar++;
if (*fCurrentChar == 0) {
return fCurrentToken = Token("", 0, _CurrentPos(),
TOKEN_END_OF_LINE);
}
bool decimal = *fCurrentChar == '.';
if (decimal || isdigit(*fCurrentChar)) {
if (*fCurrentChar == '0' && fCurrentChar[1] == 'x')
return _ParseHexOperand();
BString temp;
const char* begin = fCurrentChar;
// optional digits before the comma
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
// optional post decimal part
// (required if there are no digits before the decimal)
if (*fCurrentChar == '.') {
decimal = true;
temp << '.';
fCurrentChar++;
// optional post decimal digits
while (isdigit(*fCurrentChar)) {
temp << *fCurrentChar;
fCurrentChar++;
}
}
int32 length = fCurrentChar - begin;
if (length == 1 && decimal) {
// check for . operator
fCurrentChar = begin;
if (!_ParseOperator())
throw ParseException("unexpected character", _CurrentPos());
return fCurrentToken;
}
BString test = temp;
test << "&_";
double value;
char t[2];
int32 matches = sscanf(test.String(), "%lf&%s", &value, t);
if (matches != 2)
throw ParseException("error in constant", _CurrentPos() - length);
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_CONSTANT);
if (decimal)
fCurrentToken.value.SetTo(value);
else
fCurrentToken.value.SetTo((int64)strtoll(temp.String(), NULL, 10));
} else if (isalpha(*fCurrentChar) || *fCurrentChar == '_') {
const char* begin = fCurrentChar;
while (*fCurrentChar != 0 && (isalpha(*fCurrentChar)
|| isdigit(*fCurrentChar) || *fCurrentChar == '_')) {
fCurrentChar++;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_IDENTIFIER);
} else if (*fCurrentChar == '"' || *fCurrentChar == '\'') {
const char* begin = fCurrentChar++;
while (*fCurrentChar != 0) {
if (*fCurrentChar == '\\') {
if (*(fCurrentChar++) != 0)
fCurrentChar++;
} else if (*(fCurrentChar++) == *begin)
break;
}
int32 length = fCurrentChar - begin;
fCurrentToken = Token(begin, length, _CurrentPos() - length,
TOKEN_STRING_LITERAL);
} else {
if (!_ParseOperator()) {
int32 type = TOKEN_NONE;
switch (*fCurrentChar) {
case '\n':
type = TOKEN_END_OF_LINE;
//.........这里部分代码省略.........