本文整理汇总了C++中Tokens类的典型用法代码示例。如果您正苦于以下问题:C++ Tokens类的具体用法?C++ Tokens怎么用?C++ Tokens使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了Tokens类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的C++代码示例。
示例1: get_tokens
Tokens get_tokens( const String& s ) {
Tokens res;
char prev = ' ';
String tmp;
for ( auto c : s ) {
if ( prev != ' ' && prev != c ) {
res.push_back(tmp);
prev = c;
tmp = "";
tmp += c;
} else {
prev = c;
tmp += c;
}
}
if ( tmp != "" )
res.push_back(tmp);
return res;
}
示例2: StrSplit
Tokens StrSplit(const std::string& src, const std::string& sep)
{
Tokens r;
std::string s;
for (char i : src)
{
if (sep.find(i) != std::string::npos)
{
if (s.length()) r.push_back(s);
s.clear();
}
else
{
s += i;
}
}
if (s.length()) r.push_back(s);
return r;
}
示例3: StrSplit
Tokens StrSplit(const std::string& src, const std::string& sep)
{
Tokens r;
std::string s;
for (std::string::const_iterator i = src.begin(); i != src.end(); ++i)
{
if (sep.find(*i) != std::string::npos)
{
if (s.length()) r.push_back(s);
s = "";
}
else
{
s += *i;
}
}
if (s.length()) r.push_back(s);
return r;
}
示例4: AddChildren
static void AddChildren(Tokens& tok, TreeNode* parent)
{
TreeNode* cur = 0;
while (tok.Good())
{
std::string str;
tok >> str;
if (!tok.Good() || str.at(0) == '}')
return;
if (str.at(0) == '{')
{
if (!cur)
cur = parent->AddChild(TreeNode::strdup(""));
AddChildren(tok, cur);
continue;
}
cur = parent->AddChild(TreeNode::strdup(str));
}
}
示例5: parseDump
//static
void Command::parseDump( Tokens& tokens, bool helpMode, Command*& result )
{
if (tokens.size() < 3 && !helpMode) THROW( CWDB_ERR_PARSER, "Bad syntax: missing options in \"dump\" command" );
switch( checkAngen( tokens, 1) ) {
case ANGEN_COLLECTION: result = new DumpCollectionCommand; break;
default: THROW( CWDB_ERR_PARSER, "Bad syntax: unrecognized option in \"dump\" command" );
}
if (result != 0 && !helpMode) result->parse( tokens );
}
示例6: textCursor
void Editor::autoComplete(const QString& item)
{
if (!m_isAutoCompletionEnabled || item.isEmpty())
return;
const int currentPosition = textCursor().position();
const QString subtext = text().left(currentPosition);
const Tokens tokens = m_evaluator->scan(subtext);
if (!tokens.valid() || tokens.count() < 1)
return;
const Token lastToken = tokens.at(tokens.count() - 1);
if (!lastToken.isIdentifier())
return;
const QStringList str = item.split(':');
blockSignals(true);
QTextCursor cursor = textCursor();
cursor.setPosition(lastToken.pos());
cursor.setPosition(lastToken.pos() + lastToken.text().length(), QTextCursor::KeepAnchor);
setTextCursor(cursor);
insert(str.at(0));
blockSignals(false);
cursor = textCursor();
bool hasParensAlready;
if ((hasParensAlready = cursor.movePosition(QTextCursor::NextCharacter, QTextCursor::KeepAnchor))) {
QString nextChar = cursor.selectedText();
hasParensAlready = (nextChar == "(");
}
bool shouldAutoInsertParens = (FunctionRepo::instance()->find(str.at(0))
|| m_evaluator->hasUserFunction(str.at(0)))
&& !hasParensAlready;
if (shouldAutoInsertParens) {
insert(QString::fromLatin1("()"));
cursor = textCursor();
cursor.movePosition(QTextCursor::PreviousCharacter);
setTextCursor(cursor);
}
}
示例7: parse
void RenameMetricCommand::parse(Tokens& tokens)
{
if (tokens.size() != 5) THROW("Invalid RENAME METRIC command");
if (tokens[2].m_angen != ANGEN_IDENT) THROW("Expected existing metric name");
m_oldName = tokens[2].m_val.m_str;
if (!checkString(tokens[3], "to")) THROW("Expected key word TO");
if (tokens[4].m_angen != ANGEN_IDENT) THROW("Expected new metric name");
m_newName = tokens[4].m_val.m_str;
}
示例8: parse
void RenameDataspaceCommand::parse(Tokens& tokens)
{
if (tokens.size() != 5) THROW("Invalid RENAME DATASPACE command");
if (tokens[2].m_angen != ANGEN_IDENT) THROW("Expected existing dataspace name");
m_oldName = tokens[2].m_val.m_str;
if (!checkString(tokens[3], "to")) THROW("Expected key word TO");
if (tokens[4].m_angen != ANGEN_IDENT) THROW("Expected new dataspace name");
m_newName = tokens[4].m_val.m_str;
}
示例9: tokenize
StringUtils::Tokens StringUtils::tokenize(const std::string& str, const std::string& delimiters)
{
Tokens tokens;
std::string::size_type delimPos = 0, tokenPos = 0, pos = 0;
if(str.length()<1) return tokens;
while(1)
{
delimPos = str.find_first_of(delimiters, pos);
tokenPos = str.find_first_not_of(delimiters, pos);
if (tokenPos != std::string::npos && str[tokenPos]=='\"')
{
delimPos = str.find_first_of("\"", tokenPos+1);
pos++;
}
if(std::string::npos != delimPos)
{
if(std::string::npos != tokenPos)
{
if(tokenPos<delimPos)
{
std::string token = str.substr(pos,delimPos-pos);
if (token.length()) tokens.push_back(token);
}
}
pos = delimPos+1;
}
else
{
if(std::string::npos != tokenPos)
{
std::string token = str.substr(pos);
if (token.length()) tokens.push_back(token);
}
break;
}
}
return tokens;
}
示例10: main
int main()
{
Code code("//bla bli blo\nflap");
CodeRange cr(code);
Tokens tokens;
tokens.push_back(Token::Ptr(new Symbol(reduce(cr, 1))));
tokens.push_back(Token::Ptr(new Symbol(reduce(cr, 1))));
tokens.push_back(Token::Ptr(new Name(reduce(cr, 3))));
tokens.push_back(Token::Ptr(new Whitespace(reduce(cr, 1))));
tokens.push_back(Token::Ptr(new Name(reduce(cr, 3))));
tokens.push_back(Token::Ptr(new Whitespace(reduce(cr, 1))));
tokens.push_back(Token::Ptr(new Name(reduce(cr, 3))));
tokens.push_back(Token::Ptr(new Newline(reduce(cr, 1))));
tokens.push_back(Token::Ptr(new Name(reduce(cr, 4))));
TokenRange tr(tokens);
auto comment = Comment::construct(tr);
return 0;
}
示例11: spansToTokens
inline void spansToTokens(std::string const& str, TokenSpans const& spans, Tokens &tokens) {
if (str.empty()) return;
char const* s = &str[0];
unsigned i = 0, n = spans.size();
tokens.resize(n);
for (; i < n; ++i) {
TokenSpan const& span = spans[i];
assert(span.first < str.size());
assert(span.second <= str.size());
tokens[i].assign(s + span.first, s + span.second);
}
}
示例12: StrSplit
void RealmList::UpdateRealm( uint32 ID, const std::string& name, const std::string& address, uint32 port, uint8 icon, RealmFlags realmflags, uint8 timezone, AccountTypes allowedSecurityLevel, float popu, const char* builds)
{
// Create new if not exist or update existed
Realm& realm = m_realms[name];
realm.m_ID = ID;
realm.icon = icon;
realm.realmflags = realmflags;
realm.timezone = timezone;
realm.allowedSecurityLevel = allowedSecurityLevel;
realm.populationLevel = popu;
Tokens tokens = StrSplit(builds, " ");
Tokens::iterator iter;
for (iter = tokens.begin(); iter != tokens.end(); ++iter)
{
uint32 build = atol((*iter).c_str());
realm.realmbuilds.insert(build);
}
uint16 first_build = !realm.realmbuilds.empty() ? *realm.realmbuilds.begin() : 0;
realm.realmBuildInfo.build = first_build;
realm.realmBuildInfo.major_version = 0;
realm.realmBuildInfo.minor_version = 0;
realm.realmBuildInfo.bugfix_version = 0;
realm.realmBuildInfo.hotfix_version = ' ';
if (first_build)
if (RealmBuildInfo const* bInfo = FindBuildInfo(first_build))
if (bInfo->build == first_build)
realm.realmBuildInfo = *bInfo;
// Append port to IP address.
std::ostringstream ss;
ss << address << ":" << port;
realm.address = ss.str();
}
示例13: parseDrop
//static
void Command::parseDrop( Tokens& tokens, bool helpMode, Command*& result )
{
if (tokens.size() < 3 && !helpMode) THROW( CWDB_ERR_PARSER, "Bad syntax: missing options in \"drop\" command" );
switch( checkAngen( tokens, 1) ) {
case ANGEN_DATASPACE: result = new DropDataspaceCommand; break;
case ANGEN_COLLECTION: result = new DropCollectionCommand; break;
case ANGEN_METRIC: result = new DropMetricCommand; break;
default: THROW( CWDB_ERR_PARSER, "Bad syntax: unrecongized option in \"drop\" command" );
}
if (result != 0 && !helpMode) result->parse( tokens );
}
示例14: autoCalcSelection
void Editor::autoCalcSelection()
{
if (!m_isAutoCalcEnabled)
return;
const QString str = m_evaluator->autoFix(textCursor().selectedText());
if (str.isEmpty())
return;
// Very short (just one token) and still no calculation, then skip.
if (!m_isAnsAvailable) {
const Tokens tokens = m_evaluator->scan(text());
if (tokens.count() < 2)
return;
}
// Too short even after autofix? Don't bother either.
const Tokens tokens = m_evaluator->scan(str);
if (tokens.count() < 2)
return;
// Same reason as above, do not update "ans".
m_evaluator->setExpression(str);
const HNumber num = m_evaluator->evalNoAssign();
if (m_evaluator->error().isEmpty()) {
if (num.isNan() && m_evaluator->isUserFunctionAssign()) {
// Result is not always available when assigning a user function.
const QString message = tr("Selection result: n/a");
emit autoCalcEnabled(message);
} else {
const QString message = tr("Selection result: <b>%1</b>").arg(NumberFormatter::format(num));
emit autoCalcEnabled(message);
}
} else
emit autoCalcEnabled(m_evaluator->error());
}
示例15: tokenize
void Sentence::tokenize()
{
Tokens tokens;
::tokenize(sent_,tokens);
int i,n = tokens.size();
Syllable sy;
sy.span = 1;
sy.sent_ = this;
sy.start = 0;
for (i = 0;i < n;i ++) {
if (tokens[i].is_token) {
/*
char *viet_token = viet_to_viscii(tokens[i].value.c_str());
if (!viet_token) {
sy.id = get_sarch()[tokens[i].value];
sy.cid = get_sarch()[string("6")+tokens[i].value];
syllables.push_back(sy);
} else {
*/
const char *viet_token = tokens[i].value.c_str();
int jj,nn = strlen(viet_token);
for (jj = 0;jj < nn;jj ++)
if (viet_isalpha((unsigned char)viet_token[jj]) || viet_isdigit((unsigned char)viet_token[jj])) {
string s = viet_token;
sy.id = get_ngram()[s];
sy.cid = get_ngram()[get_std_syllable(s)];
syllables.push_back(sy);
break;
}
/*}*/
}
sy.start += tokens[i].value.size();
}
}