C++ Tokens类代码示例

 Tokens get_tokens( const String& s ) {
   Tokens res;
   char prev = ' ';
   String tmp;
   for ( auto c : s ) {
     if ( prev != ' ' && prev != c ) {
       res.push_back(tmp);
       prev = c;
       tmp = "";
       tmp += c;
     } else {
       prev = c;
       tmp += c;
     }
   }
   if ( tmp != "" )
     res.push_back(tmp);
   return res;
 }

Tokens StrSplit(const std::string& src, const std::string& sep)
{
    Tokens r;
    std::string s;
    for (char i : src)
    {
        if (sep.find(i) != std::string::npos)
        {
            if (s.length()) r.push_back(s);
            s.clear();
        }
        else
        {
            s += i;
        }
    }
    if (s.length()) r.push_back(s);
    return r;
}

Tokens StrSplit(const std::string& src, const std::string& sep)
{
    Tokens r;
    std::string s;
    for (std::string::const_iterator i = src.begin(); i != src.end(); ++i)
    {
        if (sep.find(*i) != std::string::npos)
        {
            if (s.length()) r.push_back(s);
            s = "";
        }
        else
        {
            s += *i;
        }
    }
    if (s.length()) r.push_back(s);
    return r;
}

static void AddChildren(Tokens& tok, TreeNode* parent)
{
	TreeNode* cur = 0;
	while (tok.Good())
	{
		std::string str;
		tok >> str;
		if (!tok.Good() || str.at(0) == '}')
			return;
		if (str.at(0) == '{')
		{
			if (!cur)
				cur = parent->AddChild(TreeNode::strdup(""));
			AddChildren(tok, cur);
			continue;
		}

		cur = parent->AddChild(TreeNode::strdup(str));
	}
}

//static
void Command::parseDump( Tokens& tokens, bool helpMode, Command*& result )
{
    if (tokens.size() < 3 && !helpMode) THROW( CWDB_ERR_PARSER, "Bad syntax: missing options in \"dump\" command" );

    switch( checkAngen( tokens, 1) ) {
        case ANGEN_COLLECTION: result = new DumpCollectionCommand; break;
        default: THROW( CWDB_ERR_PARSER, "Bad syntax: unrecognized option in \"dump\" command" );
    }

    if (result != 0 && !helpMode) result->parse( tokens );
}

void Editor::autoComplete(const QString& item)
{
    if (!m_isAutoCompletionEnabled || item.isEmpty())
        return;

    const int currentPosition = textCursor().position();
    const QString subtext = text().left(currentPosition);
    const Tokens tokens = m_evaluator->scan(subtext);
    if (!tokens.valid() || tokens.count() < 1)
        return;

    const Token lastToken = tokens.at(tokens.count() - 1);
    if (!lastToken.isIdentifier())
        return;

    const QStringList str = item.split(':');

    blockSignals(true);
    QTextCursor cursor = textCursor();
    cursor.setPosition(lastToken.pos());
    cursor.setPosition(lastToken.pos() + lastToken.text().length(), QTextCursor::KeepAnchor);
    setTextCursor(cursor);
    insert(str.at(0));
    blockSignals(false);

    cursor = textCursor();
    bool hasParensAlready;
    if ((hasParensAlready = cursor.movePosition(QTextCursor::NextCharacter, QTextCursor::KeepAnchor))) {
        QString nextChar = cursor.selectedText();
        hasParensAlready = (nextChar == "(");
    }
    bool shouldAutoInsertParens = (FunctionRepo::instance()->find(str.at(0))
                                   || m_evaluator->hasUserFunction(str.at(0)))
                                  && !hasParensAlready;
    if (shouldAutoInsertParens) {
        insert(QString::fromLatin1("()"));
        cursor = textCursor();
        cursor.movePosition(QTextCursor::PreviousCharacter);
        setTextCursor(cursor);
    }
}

void RenameMetricCommand::parse(Tokens& tokens)
{
    if (tokens.size() != 5) THROW("Invalid RENAME METRIC command");

    if (tokens[2].m_angen != ANGEN_IDENT) THROW("Expected existing metric name");
    m_oldName = tokens[2].m_val.m_str;

    if (!checkString(tokens[3], "to")) THROW("Expected key word TO");

    if (tokens[4].m_angen != ANGEN_IDENT) THROW("Expected new metric name");
    m_newName = tokens[4].m_val.m_str;
}

void RenameDataspaceCommand::parse(Tokens& tokens)
{
    if (tokens.size() != 5) THROW("Invalid RENAME DATASPACE command");

    if (tokens[2].m_angen != ANGEN_IDENT) THROW("Expected existing dataspace name");
    m_oldName = tokens[2].m_val.m_str;

    if (!checkString(tokens[3], "to")) THROW("Expected key word TO");

    if (tokens[4].m_angen != ANGEN_IDENT) THROW("Expected new dataspace name");
    m_newName = tokens[4].m_val.m_str;
}

StringUtils::Tokens StringUtils::tokenize(const std::string& str, const std::string& delimiters)
{
	Tokens tokens;
    std::string::size_type delimPos = 0, tokenPos = 0, pos = 0;

	if(str.length()<1)  return tokens;
	while(1)
	{
		delimPos = str.find_first_of(delimiters, pos);
		tokenPos = str.find_first_not_of(delimiters, pos);
		if (tokenPos != std::string::npos && str[tokenPos]=='\"')
		{
			delimPos = str.find_first_of("\"", tokenPos+1);
			pos++;
		}

		if(std::string::npos != delimPos)
		{
			if(std::string::npos != tokenPos)
			{
				if(tokenPos<delimPos)
				{
					std::string token = str.substr(pos,delimPos-pos);
					if (token.length()) tokens.push_back(token);
				}
			}
			pos = delimPos+1;
		}
		else
		{
			if(std::string::npos != tokenPos)
			{
				std::string token = str.substr(pos);
				if (token.length()) tokens.push_back(token);
			}
			break;
		}
	}
	return tokens;
}

int main()
{
    Code code("//bla bli blo\nflap");
    CodeRange cr(code);
    Tokens tokens;
    tokens.push_back(Token::Ptr(new Symbol(reduce(cr, 1))));
    tokens.push_back(Token::Ptr(new Symbol(reduce(cr, 1))));
    tokens.push_back(Token::Ptr(new Name(reduce(cr, 3))));
    tokens.push_back(Token::Ptr(new Whitespace(reduce(cr, 1))));
    tokens.push_back(Token::Ptr(new Name(reduce(cr, 3))));
    tokens.push_back(Token::Ptr(new Whitespace(reduce(cr, 1))));
    tokens.push_back(Token::Ptr(new Name(reduce(cr, 3))));
    tokens.push_back(Token::Ptr(new Newline(reduce(cr, 1))));
    tokens.push_back(Token::Ptr(new Name(reduce(cr, 4))));
    TokenRange tr(tokens);
    auto comment = Comment::construct(tr);
    return 0;
}

inline void spansToTokens(std::string const& str, TokenSpans const& spans, Tokens &tokens) {
  if (str.empty()) return;
  char const* s = &str[0];
  unsigned i = 0, n = spans.size();
  tokens.resize(n);
  for (; i < n; ++i) {
    TokenSpan const& span = spans[i];
    assert(span.first < str.size());
    assert(span.second <= str.size());
    tokens[i].assign(s + span.first, s + span.second);
  }

}

void RealmList::UpdateRealm( uint32 ID, const std::string& name, const std::string& address, uint32 port, uint8 icon, RealmFlags realmflags, uint8 timezone, AccountTypes allowedSecurityLevel, float popu, const char* builds)
{
    // Create new if not exist or update existed
    Realm& realm = m_realms[name];

    realm.m_ID       = ID;
    realm.icon       = icon;
    realm.realmflags = realmflags;
    realm.timezone   = timezone;
    realm.allowedSecurityLevel = allowedSecurityLevel;
    realm.populationLevel      = popu;

    Tokens tokens = StrSplit(builds, " ");
    Tokens::iterator iter;

    for (iter = tokens.begin(); iter != tokens.end(); ++iter)
    {
        uint32 build = atol((*iter).c_str());
        realm.realmbuilds.insert(build);
    }

    uint16 first_build = !realm.realmbuilds.empty() ? *realm.realmbuilds.begin() : 0;

    realm.realmBuildInfo.build = first_build;
    realm.realmBuildInfo.major_version = 0;
    realm.realmBuildInfo.minor_version = 0;
    realm.realmBuildInfo.bugfix_version = 0;
    realm.realmBuildInfo.hotfix_version = ' ';

    if (first_build)
        if (RealmBuildInfo const* bInfo = FindBuildInfo(first_build))
            if (bInfo->build == first_build)
                realm.realmBuildInfo = *bInfo;

    // Append port to IP address.
    std::ostringstream ss;
    ss << address << ":" << port;
    realm.address   = ss.str();
}

//static
void Command::parseDrop( Tokens& tokens, bool helpMode, Command*& result )
{
    if (tokens.size() < 3 && !helpMode) THROW( CWDB_ERR_PARSER, "Bad syntax: missing options in \"drop\" command" );

    switch( checkAngen( tokens, 1) ) {
        case ANGEN_DATASPACE:  result = new DropDataspaceCommand; break;
        case ANGEN_COLLECTION: result = new DropCollectionCommand; break;
        case ANGEN_METRIC:     result = new DropMetricCommand; break;
        default: THROW( CWDB_ERR_PARSER, "Bad syntax: unrecongized option in \"drop\" command" );
    }

    if (result != 0 && !helpMode) result->parse( tokens );
}

void Editor::autoCalcSelection()
{
    if (!m_isAutoCalcEnabled)
        return;

    const QString str = m_evaluator->autoFix(textCursor().selectedText());
    if (str.isEmpty())
        return;

    // Very short (just one token) and still no calculation, then skip.
    if (!m_isAnsAvailable) {
        const Tokens tokens = m_evaluator->scan(text());
        if (tokens.count() < 2)
            return;
    }

    // Too short even after autofix? Don't bother either.
    const Tokens tokens = m_evaluator->scan(str);
    if (tokens.count() < 2)
        return;

    // Same reason as above, do not update "ans".
    m_evaluator->setExpression(str);
    const HNumber num = m_evaluator->evalNoAssign();

    if (m_evaluator->error().isEmpty()) {
        if (num.isNan() && m_evaluator->isUserFunctionAssign()) {
            // Result is not always available when assigning a user function.
            const QString message = tr("Selection result: n/a");
            emit autoCalcEnabled(message);
        } else {
            const QString message = tr("Selection result: <b>%1</b>").arg(NumberFormatter::format(num));
            emit autoCalcEnabled(message);
        }
    } else
        emit autoCalcEnabled(m_evaluator->error());
}

void Sentence::tokenize()
{
	Tokens tokens;
	::tokenize(sent_,tokens);

	int i,n = tokens.size();

	Syllable sy;
	sy.span = 1;
	sy.sent_ = this;
	sy.start = 0;
	
	for (i = 0;i < n;i ++) {
		if (tokens[i].is_token) {
			/*
				char *viet_token = viet_to_viscii(tokens[i].value.c_str());
				if (!viet_token) {
				sy.id = get_sarch()[tokens[i].value];
				sy.cid = get_sarch()[string("6")+tokens[i].value];
				syllables.push_back(sy);
				} else {
			*/
			const char *viet_token = tokens[i].value.c_str();
			int jj,nn = strlen(viet_token);
			for (jj = 0;jj < nn;jj ++)
				if (viet_isalpha((unsigned char)viet_token[jj]) || viet_isdigit((unsigned char)viet_token[jj])) {
					string s = viet_token;
					sy.id = get_ngram()[s];
					sy.cid = get_ngram()[get_std_syllable(s)];
					syllables.push_back(sy);
					break;
				}
			/*}*/
		}
		sy.start += tokens[i].value.size();
	}
}