C++ token类代码示例

void Foam::primitiveEntry::append
(
    const token& currToken,
    const dictionary& dict,
    Istream& is
)
{
    if (currToken.isWord())
    {
        const word& w = currToken.wordToken();

        if
        (
            w.size() == 1
         || (
                !(w[0] == '$' && expandVariable(w, dict))
             && !(w[0] == '#' && expandFunction(w, dict, is))
            )
        )
        {
            newElmt(tokenIndex()++) = currToken;
        }
    }
    else
    {
        newElmt(tokenIndex()++) = currToken;
    }
}

/*
 * Returns the contents of the lexer as a token
 */
bool lexer::get_token(token &tok) {

	// set appropriate values
	tok.set_text(text);
	tok.set_type(type);
	return true;
}

    void intersect (basic_charset &rhs_, basic_charset &overlap_)
    {
        _token.intersect (rhs_._token, overlap_._token);

        if (!overlap_._token.empty ())
        {
            typename index_set::const_iterator iter_ = _index_set.begin ();
            typename index_set::const_iterator end_ = _index_set.end ();

            for (; iter_ != end_; ++iter_)
            {
                overlap_._index_set.insert (*iter_);
            }

            iter_ = rhs_._index_set.begin ();
            end_ = rhs_._index_set.end ();

            for (; iter_ != end_; ++iter_)
            {
                overlap_._index_set.insert (*iter_);
            }

            if (_token.empty ())
            {
                _index_set.clear ();
            }

            if (rhs_._token.empty ())
            {
                rhs_._index_set.clear ();
            }
        }
    }

int
opPriority(token t)
{
    if ( (binOP_Table.end() != binOP_Table.find(t.Tok())) )
	return binOP_Table[t.Tok()];
    else // this will stop OpPrecedence parsing once we hit a
	return -1;  // non-op while evaluating InfixEpxr
}

bool token::operator == (const token& rhs) const
{
   if (this == &rhs)
      return true;

   if (valid() && rhs.valid())
      return to_string() == rhs.to_string();
   else
      return false;
}

  virtual parse_result parse( string::const_iterator& i, 
			      string::const_iterator e, 
			      token& tok ) const
  {
    string::const_iterator j = i;
    if ( *j != '&' && *j != '+' ) return failed; ++j;
    while ( j < e && ( *j == '.' || *j == '-' || isalnum(*j) ) ) ++j;
    tok.assign(i, j); tok.type( tok_types::pn_lit ); i = j;
    return done;
  }

/*
 * Returns the contents at the root
 */
bool syn_tree::get_root_contents(token &tok) {

	// check that root token exists
	if(!root)
		return false;

	// set the text & type
	tok.set_text(root->get_text());
	tok.set_type(root->get_type());
	return true;
}

zstring::zstring(const token& tok)
    : _buf(0)
    , _pool(0)
{
    if(tok.len() == 0)
        _zptr = _zend = nullstring;
    else {
        _zptr = tok.ptr();
        _zend = tok.ptre() - 1;
    }
}

/*
 * Set the contents of the current token
 */
bool syn_tree::set_contents(token &tok) {

	// make sure current token exists
	if(!cur)
		return false;

	// set current token contents
	cur->set_text(tok.get_text());
	cur->set_type(tok.get_type());
	return true;
}

/*
 * Returns contents at current token
 */
bool syn_tree::get_contents(token &tok) {

	// check that current token exists
	if(!cur)
		return false;

	// set the text & type
	tok.set_text(cur->get_text());
	tok.set_type(cur->get_type());
	return true;
}

  static bool is_greater_precedence(const token &one, const token &two) {
    size_t one_id = one.first_matching_sym_op_fn_id();
    size_t two_id = two.first_matching_sym_op_fn_id();
    typename Fn_Table::fn_info one_info = Fn_Table::get_info(one_id);
    typename Fn_Table::fn_info two_info = Fn_Table::get_info(two_id);
    bool one_is_left_assoc = one_info.is_left_assoc();
    size_t one_precedence = one_info.precedence();
    size_t two_precedence = two_info.precedence();

    return ((one_is_left_assoc && (one_precedence >= two_precedence))
            || (!one_is_left_assoc && (one_precedence > two_precedence)));
  }

 void dotfile_visitor::format_node(abstract_node *node, const char *name, const token &tok)
 {
   if (!m_connectTo)
   {
     if (tok.type() == token_types::STRING_LITERAL)
       fprintf(m_file, "\tptr%p [label=\"[%s]\\n[%s]\\n\\\"%.*s\\\"\"];\n", node, name, token_types::names[tok.type()], tok.length() - 2, tok.text() + 1);
     else if (tok.text())
       fprintf(m_file, "\tptr%p [label=\"[%s]\\n[%s]\\n%.*s\"];\n", node, name, token_types::names[tok.type()], tok.length(), tok.text());
     else
       format_node(node, name);
   }
 }

string symbol_table::new_const(token tkconst)
{
    string type = type_of(tkconst);
    auto ent = consts_.find(type+tkconst.text());
    if (ent != consts_.end()) {
        return ent->second.place();
    }
    
    const_desc desc(tkconst);
    consts_[type+tkconst.text()] = desc;
    return desc.place();
}

static inline token hash(const std::string &pw, token const& salt)
{
	token result;
	if(::libscrypt_scrypt(
		(const uint8_t*)pw.data(), pw.size(),
		salt.data(), salt.size(),
		SCRYPT_N, SCRYPT_r, SCRYPT_p,
		result.data(), result.size()
	) == -1)
		throw std::runtime_error("Incapable of hashing plaintext");

	return result;
}

/*
 * Sets the contents of the specified child token of current token
 */
bool syn_tree::set_child_contents(token &tok, unsigned int index) {

	// check to make sure child token exists
	if(!cur
			|| !get_size()
			|| index >= get_size())
		return false;

	// set contents
	token *child = cur->get_child(index);
	child->set_text(tok.get_text());
	child->set_type(tok.get_type());
	return true;
}