C++ TokenList类代码示例

//Remove all block comments from the tokenList including /* and */ markers
//Returns the number of block comments removed (where each pair of /* and */ count as one comment)
int removeBlockComments(TokenList &tokenList) {
	//this funtion deletes the block comments in the tokenlist
	int count = 0;
	bool deleted = false;
	Token* temp, *destroy;
	temp = tokenList.getFirst();
	while (temp) {
		//loop check if list is empty
		deleted = false;
		if (temp->getStringRep() == "/*") {
			//upon finding a block entry comment you keep deleating tokens till you find the exit token
			count++;
			while (!deleted) {
				destroy = temp;
				temp = temp->getNext();
				tokenList.deleteToken(destroy);
				if (temp->getStringRep() == "*/") {
					//once the exit block token is found stop delete looping and continue searching through the list for block entry symbols
					destroy = temp;
					temp = temp->getNext();
					deleted = true;
					tokenList.deleteToken(destroy);
				}
			}
		}
		else {
			temp = temp->getNext();
		}
	}
	return count;
}

unsigned int Bayes::GetTokenCount(const string& token, BayesTokenType type)
{
	TokenList list = type == Spam ? spam : type == Ham ? ham : TokenList();
	if(list.count(token) == 0)
		return 0;
	return list[token];
}

Expression ExpressionBuilder::Build(std::string expression) {
	TokenList tokens = m_tokenizer(expression);

	TokenList postfix = m_converter(tokens);

	std::stack<ExpressionNode*> nodeStk;

	auto it = postfix.begin();

	nodeStk.push(m_nodeBuilder.NewExpressionNode(*it));

	while (++it != postfix.end()) {
		ExpressionNode* node = m_nodeBuilder.NewExpressionNode(*it);

		switch (node->NumChildren()) {
		case 0:
			nodeStk.push(node);
			break;
		case 1:
			static_cast<OperatorNode*>(node)->SetChild(nodeStk.top());
			nodeStk.pop();
			nodeStk.push(node);
			break;
		case 2:
			static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 1);
			nodeStk.pop();
			static_cast<OperatorNode*>(node)->SetChild(nodeStk.top(), 0);
			nodeStk.pop();
			nodeStk.push(node);
			break;
		}
	}

	return{ nodeStk.top(),  m_nodeBuilder.GetVariableMap() };
}

Declaration* CssParser::parseDeclaration () {
  Declaration* declaration = NULL;
  TokenList property;

  if (!parseProperty(&property))
    return NULL;
  
  skipWhitespace();

  declaration = new Declaration(property.toString());
  
  if (tokenizer->getTokenType() != Token::COLON) {
    throw new ParseException(tokenizer->getToken()->str,
                             "colon following property(':')");
  }
  tokenizer->readNextToken();
  skipWhitespace();

  TokenList* value = parseValue();
  if (value == NULL) {
    throw new ParseException(tokenizer->getToken()->str,
                             "value for property");
  }
  declaration->setValue(value);
  return declaration;
}

//Removes all comments from the tokenList including the -- marker
//Returns the number of comments removed
int removeComments(TokenList &tokenList)
{
	int count = 0;
	Token* current = tokenList.getFirst();//initialized current to the first token

		while (current) //this loop run till end of the list
		{

			if (current->getStringRep() == "--")
			{
				count++; //count comment to remove
				Token* remove_first = current;//make a pointer to remove '--'

				if (current->getNext() != NULL)
				{
					Token* remove_second = current->getNext();
					//check comment on the last line
					current = remove_second->getNext();
					tokenList.deleteToken(remove_first);
					tokenList.deleteToken(remove_second);
				}
				else
				{
					delete current;
				}
			}
			else
			{
				current = current->getNext();
			}
		}

	return count; //Returns the number of comments removed
}

//remove token of given type
int removeTokensOfType(TokenList &tokenList, tokenType type)
{
	int count = 0;

	Token* current = tokenList.getFirst();//initialized current to the first token

	while (current) //this loop run till end of the list
	{
		if (current->getTokenType() == type)
		{
			count++;
			Token* remove_first = current;

			if (current->getNext() != NULL)
			{
				tokenList.deleteToken(remove_first);
			}
			else
			{
				delete current;
			}
		}
		else
		{
			current = current->getNext();
		}
	}

	return count;
}

	//////////////////////////////////////////////////////////////////////////
	//
	// Class VMemory
	// 
	void ETCompiler::VMemory::LoadCode( const std::string& code )
	{
		Reset();

		std::string line;
		std::string::const_iterator iter = code.begin();

		while ( iter != code.end() )
		{
			line += *iter;

			if ( *iter == '\n' )
			{
				//TranslateLine(line);
				TokenList tokList = str2Tok( line );

				Instruction ins;

				for ( auto iter = tokList.begin(); iter != tokList.end(); iter++ )
				{
					ins.m_elements.push_back( *iter );
				}

				m_instructions.push_back( ins );

				line = "";
			}

			iter++;
		}
	}

//Creates a new TokenList, and returns a pointer to this list
//Searches for all conditional expressions in tokenList and appends them to the new list
//Format is as follows:
//Each token that is part of a condtional expression is appended sequentially
//At the end of a conditional expression a newline character is appened
//Example: if (a = true) then
//Your list should include "(", "a", "=", "true", ")" and "\n"
//tokenList is NOT modified
TokenList* findAllConditionalExpressions(const TokenList &tokenList)
{
    TokenList* conditionalExpressionTokenList = new TokenList();
    Token* t = tokenList.getFirst();
    while (t!= nullptr && t->getNext()!= nullptr)
    {
        if (t!= nullptr && (t->getStringRep() == "if" || t->getStringRep() == "elsif") && t->getPrev()->getStringRep()!="end" && !t->isComment()&& t->getNext()!= nullptr)
        {
            t = t->getNext();
            while (t!= nullptr && !(t->getStringRep() == "then" && !t->isComment()) && t->getNext()!= nullptr){
                conditionalExpressionTokenList->append(t->getStringRep());
                t = t->getNext();}
            conditionalExpressionTokenList ->append("\n");
        }
        if (t!= nullptr && t->getStringRep() == "when" && !t->isComment() && t->getNext()!= nullptr)
        {
            t = t->getNext();
            while (t!=nullptr && !(t->getStringRep() == "else" && !t->isComment())&& t->getNext()!= nullptr)
            {
                conditionalExpressionTokenList->append(t->getStringRep());
                t = t->getNext();}
            conditionalExpressionTokenList ->append("\n");
        }
    t = t->getNext();}
    return conditionalExpressionTokenList;
}

//check for mismatch type
int error_type(const TokenList &tokenList, const TokenList &tokenList2)
{
	int mis_type = 0;

	Token *current = new Token;
	Token *numbers = new Token;
	Token *temp = new Token;

	current = tokenList.getFirst();
	numbers = tokenList2.getFirst();

	while (current && numbers)
	{
	// find a operator and look at surrounding tokens
		if (current->getTokenType() == T_Identifier && current->getTokenDetails() != NULL &&
			current->getNext()->getTokenType() == T_Operator && current->getNext()->getNext()->getTokenDetails() != NULL &&
			current->getNext()->getNext()->getTokenType() != T_Literal)
		{
			string temp_typeA = current->getTokenDetails()->type;
			string temp_typeB = current->getNext()->getNext()->getTokenDetails()->type;
            //if details of type are not the same
			if (temp_typeA != temp_typeB)
			{
				cout << "ERROR mismatch type on line: " << numbers->getStringRep() << endl;
				mis_type++;
			}
		}
		numbers = numbers->getNext();
		current = current->getNext();
	}
	return mis_type;
}

string GetSection(const string& File, const string& Sec)
{
  istringstream is(File);
  ASSERT(is);

  TokenList TL;
  bool Found = false;
  do {
    is >> TL;
    if (!TL.empty()) {
      TokenList TL2(TL[0],"=\\");
      Found = TL2[0] == Sec;
    }
  } while(!Found && is);

  if (Found) { // Get the rest of the lines in the section
    while (*(TL.rbegin()->rbegin()) == '\\') {
      TokenList TL2;
      is >> TL2;
      TL += TL2;
    }
  }

  return TL.getString();
}

void ValueProcessor::interpolate(std::string &str,
                                 const ValueScope &scope) const {
  size_t start, end = 0;
  string key, value;
  const TokenList *var;
  TokenList variable;

  while ((start = str.find("@{", end)) != string::npos &&
         (end = str.find("}", start)) != string::npos) {
    key = "@";
    key.append(str.substr(start + 2, end - (start + 2)));

    var = scope.getVariable(key);

    if (var != NULL) {
      variable = *var;

      processValue(variable, scope);

      // Remove quotes off strings.
      if (variable.size() == 1 && variable.front().type == Token::STRING) {
        variable.front().removeQuotes();
      }

      value = variable.toString();

      str.replace(start, (end + 1) - start, value);
      end = start + value.length();
    }
  }
}

bool UnprocessedStatement::processDeclaration (Declaration* declaration) {
  TokenList property;
  Token keyword;

#ifdef WITH_LIBGLOG
  VLOG(3) << "Declaration";
#endif
  
  getValue(declaration->getValue());
  
  if (declaration->getValue().empty() ||
      declaration->getValue().front().type != Token::COLON) {
    return NULL;
  }
    
  declaration->getValue().pop_front();
  
  getProperty(property);
  keyword = property.front();
  keyword.assign(property.toString());
  
  declaration->setProperty(keyword);
  
  return true;
}

const TokenList *ValueProcessor::processDeepVariable(
    TokenList::const_iterator &i,
    TokenList::const_iterator &end,
    const ValueScope &scope) const {
  const TokenList *var;
  TokenList variable;
  std::string key = "@";

  if (i == end || (*i).type != Token::OTHER || (*i) != "@")
    return NULL;

  i++;

  if (i == end || (*i).type != Token::ATKEYWORD ||
      (var = scope.getVariable((*i))) == NULL) {
    i--;
    return NULL;
  }

  variable = *var;
  processValue(variable, scope);

  if (variable.size() != 1 || variable.front().type != Token::STRING) {
    i--;
    return NULL;
  }

  i++;
  // generate key with '@' + var without quotes
  variable.front().removeQuotes();
  key.append(variable.front());

  return scope.getVariable(key);
}

Variable_array * VariableScript::parse(const c8 *text, size_t length, const Token::File *file) {
    TokenList tokens;
    if (m_lexer.scan(text, length, file, tokens) < 0) {
        _makeErrorMsg("lexical error at %s", m_lexer.error().c_str());
        return 0;
    }
    tokens.push_back(vnnew Token(file));
    for (TokenList::iterator it = tokens.begin(); it != tokens.end(); ++it) {
        LR_Parser::Result ret = m_parser.input(it->ptr());
        if (ret == LR_Parser::kAccept) {
            break;
        }
        if (ret == LR_Parser::kFailed) {
            _makeErrorMsg("syntax error at %s(%u): %s", (file ? file->name.c_str() : ""), it->ptr()->row + 1, it->ptr()->text.c_str());
            break;
        }
    }
    LR_Node *root = m_parser.result();
    if (root) {
        root->grab();
        m_parser.reset();
    } else {
        m_parser.reset();
        return 0;
    }
    m_errorMsg.clear();
    Variable_array *ret = vnnew Variable_array();
    _build_value_list(root->child(0), ret);
    root->drop();
    
    return ret;
}

bool ValueProcessor::validateCondition(const TokenList &value,
                                       const ValueScope &scope,
                                       bool defaultVal) const {
  TokenList::const_iterator i = value.begin();
  TokenList::const_iterator end = value.end();
  bool negate = false;
  bool ret;

  skipWhitespace(i, end);

  if (*i == "not") {
    negate = true;
    i++;
  }
    
  ret = validateValue(i, end, scope, defaultVal);

  skipWhitespace(i, end);

  while (ret == true && i != value.end() && *i == "and") {
    i++;

    skipWhitespace(i, end);

    ret = validateValue(i, end, scope, defaultVal);

    skipWhitespace(i, end);
  }

  return negate ? !ret : ret;
}