本文整理汇总了C++中TokenList类的典型用法代码示例。如果您正苦于以下问题:C++ TokenList类的具体用法?C++ TokenList怎么用?C++ TokenList使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了TokenList类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的C++代码示例。
示例1: removeBlockComments
//Remove all block comments from the tokenList including /* and */ markers
//Returns the number of block comments removed (where each pair of /* and */ count as one comment)
int removeBlockComments(TokenList &tokenList) {
//this funtion deletes the block comments in the tokenlist
int count = 0;
bool deleted = false;
Token* temp, *destroy;
temp = tokenList.getFirst();
while (temp) {
//loop check if list is empty
deleted = false;
if (temp->getStringRep() == "/*") {
//upon finding a block entry comment you keep deleating tokens till you find the exit token
count++;
while (!deleted) {
destroy = temp;
temp = temp->getNext();
tokenList.deleteToken(destroy);
if (temp->getStringRep() == "*/") {
//once the exit block token is found stop delete looping and continue searching through the list for block entry symbols
destroy = temp;
temp = temp->getNext();
deleted = true;
tokenList.deleteToken(destroy);
}
}
}
else {
temp = temp->getNext();
}
}
return count;
}
示例2: TokenList
unsigned int Bayes::GetTokenCount(const string& token, BayesTokenType type)
{
TokenList list = type == Spam ? spam : type == Ham ? ham : TokenList();
if(list.count(token) == 0)
return 0;
return list[token];
}
示例3: m_tokenizer
Expression ExpressionBuilder::Build(std::string expression) {
TokenList tokens = m_tokenizer(expression);
TokenList postfix = m_converter(tokens);
std::stack<ExpressionNode*> nodeStk;
auto it = postfix.begin();
nodeStk.push(m_nodeBuilder.NewExpressionNode(*it));
while (++it != postfix.end()) {
ExpressionNode* node = m_nodeBuilder.NewExpressionNode(*it);
switch (node->NumChildren()) {
case 0:
nodeStk.push(node);
break;
case 1:
static_cast<OperatorNode*>(node)->SetChild(nodeStk.top());
nodeStk.pop();
nodeStk.push(node);
break;
case 2:
static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 1);
nodeStk.pop();
static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 0);
nodeStk.pop();
nodeStk.push(node);
break;
}
}
return{ nodeStk.top(), m_nodeBuilder.GetVariableMap() };
}
示例4: skipWhitespace
Declaration* CssParser::parseDeclaration () {
Declaration* declaration = NULL;
TokenList property;
if (!parseProperty(&property))
return NULL;
skipWhitespace();
declaration = new Declaration(property.toString());
if (tokenizer->getTokenType() != Token::COLON) {
throw new ParseException(tokenizer->getToken()->str,
"colon following property(':')");
}
tokenizer->readNextToken();
skipWhitespace();
TokenList* value = parseValue();
if (value == NULL) {
throw new ParseException(tokenizer->getToken()->str,
"value for property");
}
declaration->setValue(value);
return declaration;
}
示例5: removeComments
//Removes all comments from the tokenList including the -- marker
//Returns the number of comments removed
int removeComments(TokenList &tokenList)
{
int count = 0;
Token* current = tokenList.getFirst();//initialized current to the first token
while (current) //this loop run till end of the list
{
if (current->getStringRep() == "--")
{
count++; //count comment to remove
Token* remove_first = current;//make a pointer to remove '--'
if (current->getNext() != NULL)
{
Token* remove_second = current->getNext();
//check comment on the last line
current = remove_second->getNext();
tokenList.deleteToken(remove_first);
tokenList.deleteToken(remove_second);
}
else
{
delete current;
}
}
else
{
current = current->getNext();
}
}
return count; //Returns the number of comments removed
}
示例6: removeTokensOfType
//remove token of given type
int removeTokensOfType(TokenList &tokenList, tokenType type)
{
int count = 0;
Token* current = tokenList.getFirst();//initialized current to the first token
while (current) //this loop run till end of the list
{
if (current->getTokenType() == type)
{
count++;
Token* remove_first = current;
if (current->getNext() != NULL)
{
tokenList.deleteToken(remove_first);
}
else
{
delete current;
}
}
else
{
current = current->getNext();
}
}
return count;
}
示例7: Reset
//////////////////////////////////////////////////////////////////////////
//
// Class VMemory
//
void ETCompiler::VMemory::LoadCode( const std::string& code )
{
Reset();
std::string line;
std::string::const_iterator iter = code.begin();
while ( iter != code.end() )
{
line += *iter;
if ( *iter == '\n' )
{
//TranslateLine(line);
TokenList tokList = str2Tok( line );
Instruction ins;
for ( auto iter = tokList.begin(); iter != tokList.end(); iter++ )
{
ins.m_elements.push_back( *iter );
}
m_instructions.push_back( ins );
line = "";
}
iter++;
}
}
示例8: findAllConditionalExpressions
//Creates a new TokenList, and returns a pointer to this list
//Searches for all conditional expressions in tokenList and appends them to the new list
//Format is as follows:
//Each token that is part of a condtional expression is appended sequentially
//At the end of a conditional expression a newline character is appened
//Example: if (a = true) then
//Your list should include "(", "a", "=", "true", ")" and "\n"
//tokenList is NOT modified
TokenList* findAllConditionalExpressions(const TokenList &tokenList)
{
TokenList* conditionalExpressionTokenList = new TokenList();
Token* t = tokenList.getFirst();
while (t!= nullptr && t->getNext()!= nullptr)
{
if (t!= nullptr && (t->getStringRep() == "if" || t->getStringRep() == "elsif") && t->getPrev()->getStringRep()!="end" && !t->isComment()&& t->getNext()!= nullptr)
{
t = t->getNext();
while (t!= nullptr && !(t->getStringRep() == "then" && !t->isComment()) && t->getNext()!= nullptr){
conditionalExpressionTokenList->append(t->getStringRep());
t = t->getNext();}
conditionalExpressionTokenList ->append("\n");
}
if (t!= nullptr && t->getStringRep() == "when" && !t->isComment() && t->getNext()!= nullptr)
{
t = t->getNext();
while (t!=nullptr && !(t->getStringRep() == "else" && !t->isComment())&& t->getNext()!= nullptr)
{
conditionalExpressionTokenList->append(t->getStringRep());
t = t->getNext();}
conditionalExpressionTokenList ->append("\n");
}
t = t->getNext();}
return conditionalExpressionTokenList;
}
示例9: error_type
//check for mismatch type
int error_type(const TokenList &tokenList, const TokenList &tokenList2)
{
int mis_type = 0;
Token *current = new Token;
Token *numbers = new Token;
Token *temp = new Token;
current = tokenList.getFirst();
numbers = tokenList2.getFirst();
while (current && numbers)
{
// find a operator and look at surrounding tokens
if (current->getTokenType() == T_Identifier && current->getTokenDetails() != NULL &&
current->getNext()->getTokenType() == T_Operator && current->getNext()->getNext()->getTokenDetails() != NULL &&
current->getNext()->getNext()->getTokenType() != T_Literal)
{
string temp_typeA = current->getTokenDetails()->type;
string temp_typeB = current->getNext()->getNext()->getTokenDetails()->type;
//if details of type are not the same
if (temp_typeA != temp_typeB)
{
cout << "ERROR mismatch type on line: " << numbers->getStringRep() << endl;
mis_type++;
}
}
numbers = numbers->getNext();
current = current->getNext();
}
return mis_type;
}
示例10: GetSection
string GetSection(const string& File, const string& Sec)
{
istringstream is(File);
ASSERT(is);
TokenList TL;
bool Found = false;
do {
is >> TL;
if (!TL.empty()) {
TokenList TL2(TL[0],"=\\");
Found = TL2[0] == Sec;
}
} while(!Found && is);
if (Found) { // Get the rest of the lines in the section
while (*(TL.rbegin()->rbegin()) == '\\') {
TokenList TL2;
is >> TL2;
TL += TL2;
}
}
return TL.getString();
}
示例11: interpolate
void ValueProcessor::interpolate(std::string &str,
const ValueScope &scope) const {
size_t start, end = 0;
string key, value;
const TokenList *var;
TokenList variable;
while ((start = str.find("@{", end)) != string::npos &&
(end = str.find("}", start)) != string::npos) {
key = "@";
key.append(str.substr(start + 2, end - (start + 2)));
var = scope.getVariable(key);
if (var != NULL) {
variable = *var;
processValue(variable, scope);
// Remove quotes off strings.
if (variable.size() == 1 && variable.front().type == Token::STRING) {
variable.front().removeQuotes();
}
value = variable.toString();
str.replace(start, (end + 1) - start, value);
end = start + value.length();
}
}
}
示例12: processDeclaration
bool UnprocessedStatement::processDeclaration (Declaration* declaration) {
TokenList property;
Token keyword;
#ifdef WITH_LIBGLOG
VLOG(3) << "Declaration";
#endif
getValue(declaration->getValue());
if (declaration->getValue().empty() ||
declaration->getValue().front().type != Token::COLON) {
return NULL;
}
declaration->getValue().pop_front();
getProperty(property);
keyword = property.front();
keyword.assign(property.toString());
declaration->setProperty(keyword);
return true;
}
示例13: processValue
const TokenList *ValueProcessor::processDeepVariable(
TokenList::const_iterator &i,
TokenList::const_iterator &end,
const ValueScope &scope) const {
const TokenList *var;
TokenList variable;
std::string key = "@";
if (i == end || (*i).type != Token::OTHER || (*i) != "@")
return NULL;
i++;
if (i == end || (*i).type != Token::ATKEYWORD ||
(var = scope.getVariable((*i))) == NULL) {
i--;
return NULL;
}
variable = *var;
processValue(variable, scope);
if (variable.size() != 1 || variable.front().type != Token::STRING) {
i--;
return NULL;
}
i++;
// generate key with '@' + var without quotes
variable.front().removeQuotes();
key.append(variable.front());
return scope.getVariable(key);
}
示例14: _makeErrorMsg
Variable_array * VariableScript::parse(const c8 *text, size_t length, const Token::File *file) {
TokenList tokens;
if (m_lexer.scan(text, length, file, tokens) < 0) {
_makeErrorMsg("lexical error at %s", m_lexer.error().c_str());
return 0;
}
tokens.push_back(vnnew Token(file));
for (TokenList::iterator it = tokens.begin(); it != tokens.end(); ++it) {
LR_Parser::Result ret = m_parser.input(it->ptr());
if (ret == LR_Parser::kAccept) {
break;
}
if (ret == LR_Parser::kFailed) {
_makeErrorMsg("syntax error at %s(%u): %s", (file ? file->name.c_str() : ""), it->ptr()->row + 1, it->ptr()->text.c_str());
break;
}
}
LR_Node *root = m_parser.result();
if (root) {
root->grab();
m_parser.reset();
} else {
m_parser.reset();
return 0;
}
m_errorMsg.clear();
Variable_array *ret = vnnew Variable_array();
_build_value_list(root->child(0), ret);
root->drop();
return ret;
}
示例15: validateCondition
bool ValueProcessor::validateCondition(const TokenList &value,
const ValueScope &scope,
bool defaultVal) const {
TokenList::const_iterator i = value.begin();
TokenList::const_iterator end = value.end();
bool negate = false;
bool ret;
skipWhitespace(i, end);
if (*i == "not") {
negate = true;
i++;
}
ret = validateValue(i, end, scope, defaultVal);
skipWhitespace(i, end);
while (ret == true && i != value.end() && *i == "and") {
i++;
skipWhitespace(i, end);
ret = validateValue(i, end, scope, defaultVal);
skipWhitespace(i, end);
}
return negate ? !ret : ret;
}