C++ Tokenize函数代码示例

	void CAnalysisHowPage::FillMode(short t)
	{
		WBSF::StringVector MODE_NAME = Tokenize(GetString(IDS_CMN_TIME_FORMAT_DATAHEAD1), ";|");

		int curSel = m_TTModeCtrl.GetCurItemData();
		m_TTModeCtrl.ResetContent();

		for (int m = 0; m < CTM::NB_MODE; m++)
		{
			if (m_sourceTM.IsModeAvailable(CTM(t, m)))
			{
				int pos = m_TTModeCtrl.AddString(MODE_NAME[m].c_str());
				m_TTModeCtrl.SetItemData(pos, m);
			}
		}

		m_TTModeCtrl.SelectFromItemData(curSel);
	}

bool promote_dependencies(const std::string& snapshot, std::vector<std::string>& dependencies)
{
	zfs_elevate();
	
	std::cout << "Searching for origin " << snapshot << std::endl;
	
	std::string snap_data;
	int rc = exec_wait(find_zfs_cmd(), snap_data, "list", "-H", "-o", "name", NULL);
	if(rc!=0)
		return false;
	
	std::vector<std::string> snaps;
	Tokenize(snap_data, snaps, "\n");
		
	std::string snap_folder = ExtractFilePath(snapshot);
	for(size_t i=0;i<snaps.size();++i)
	{	
		if( !next(trim(snaps[i]), 0, snap_folder)
			|| trim(snaps[i]).size()<=snap_folder.size() )
			continue;
		
		std::string stdout;
		std::string subvolume_folder = snaps[i];
		int rc=exec_wait(find_zfs_cmd(), stdout, "get", "-H", "-o", "value", "origin", subvolume_folder.c_str(), NULL);
		if(rc==0)
		{
			stdout=trim(stdout);
			
			if(stdout==snapshot)
			{	
				std::cout << "Origin is " << subvolume_folder << std::endl;
				
				if(exec_wait(find_zfs_cmd(), true, "promote", subvolume_folder.c_str(), NULL)!=0)
				{
					return false;
				}
				
				dependencies.push_back(subvolume_folder);
			}
		}
	}
	
	return true;
}

/** check whether a file exists */
bool Parameter::FilesExist(const string &paramName, int fieldNo, std::vector<std::string> const& extensions)
{
  typedef std::vector<std::string> StringVec;
  StringVec::const_iterator iter;

  PARAM_MAP::const_iterator iterParam = m_setting.find(paramName);
  if (iterParam == m_setting.end()) {
    // no param. therefore nothing to check
    return true;
  }
  const StringVec &pathVec = (*iterParam).second;
  for (iter = pathVec.begin() ; iter != pathVec.end() ; ++iter) {
    StringVec vec = Tokenize(*iter);

    size_t tokenizeIndex;
    if (fieldNo == -1)
      tokenizeIndex = vec.size() - 1;
    else
      tokenizeIndex = static_cast<size_t>(fieldNo);

    if (tokenizeIndex >= vec.size()) {
      stringstream errorMsg("");
      errorMsg << "Expected at least " << (tokenizeIndex+1) << " tokens per entry in '"
               << paramName << "', but only found "
               << vec.size();
      UserMessage::Add(errorMsg.str());
      return false;
    }
    const string &pathStr = vec[tokenizeIndex];

    bool fileFound=0;
    for(size_t i=0; i<extensions.size() && !fileFound; ++i) {
      fileFound|=FileExists(pathStr + extensions[i]);
    }
    if(!fileFound) {
      stringstream errorMsg("");
      errorMsg << "File " << pathStr << " does not exist";
      UserMessage::Add(errorMsg.str());
      return false;
    }
  }
  return true;
}

// Parses a string of the form "source_planet destination_planet num_ships"
// and calls state.ExecuteOrder. If that fails, the player is dropped.
bool Game::ExecuteOrder(int playerID, const std::string& order) {
	std::vector<std::string> tokens = Tokenize(order, " ");
	if (tokens.size() != 3) return -1;
	
	int sourcePlanet = atoi(tokens[0].c_str());
	int destinationPlanet = atoi(tokens[1].c_str());
	int numShips = atoi(tokens[2].c_str());

	if(!state.ExecuteOrder(desc, playerID, sourcePlanet, destinationPlanet, numShips)) {
		WriteLogMessage("Dropping player " + to_string(playerID) +
						". source.Owner() = " + to_string(state.planets[sourcePlanet].owner) + ", playerID = " +
						to_string(playerID) + ", numShips = " + to_string(numShips) +
						", source.NumShips() = " + to_string(state.planets[sourcePlanet].numShips));
		std::cerr << "Dropping player " << playerID << " because of invalid order: " << order << std::endl;
		state.DropPlayer(playerID);
		return false;
	}
	return true;
}

void sdpPlayerParseStateTableData::GetBoundary(double TimeStampForSearch, double &TopBoundary, double &LowBoundary) {
    // Load Relative Field 
    std::vector <double> TimeIndex;    
    unsigned int i  = 0;
    std::string ExtractLine;        
    TopBoundary = LowBoundary = 0.0;
        // where is TimeStampForSearch postition in index file? 
    for ( i  = 0 ; i < Index.size(); i++) {        
        std::vector <std::string> Token;   
        Tokenize(Index.at(i).LineAtIndex, Token, Header.Delimiter);// get rid of first string
        TopBoundary =  strtod(Token.at(IndexOfTimeField).c_str(), NULL );
        if (TimeStampForSearch <= TopBoundary)           
            break;        
        LowBoundary = TopBoundary;
        Token.clear();
    } 

    return;
}

static KMETHOD Statement_namespace(KonohaContext *kctx, KonohaStack *sfp)
{
	VAR_TypeCheck(stmt, ns, reqc);
	kstatus_t result = K_CONTINUE;
	kToken *tk = SUGAR kNode_GetToken(kctx, stmt, KSymbol_BlockPattern, NULL);
	if(tk != NULL && tk->resolvedSyntaxInfo->keyword == TokenType_LazyBlock) {
		INIT_GCSTACK();
		kNameSpace *ns = new_(NameSpace, kNode_ns(stmt), _GcStack);
		KTokenSeq range = {ns, KGetParserContext(kctx)->preparedTokenList};
		KTokenSeq_Push(kctx, range);
		SUGAR Tokenize(kctx, ns, kString_text(tk->text), tk->uline, tk->indent, range.tokenList);
		KTokenSeq_End(kctx, range);
		result = SUGAR EvalTokenList(kctx, &range, NULL/*trace*/);
		KTokenSeq_Pop(kctx, range);
		RESET_GCSTACK();
		kNode_Type(kctx, stmt, KNode_Done, KType_void);
	}
	KReturnUnboxValue(result == K_CONTINUE);
}

void FeatureFunction::SetTuneableComponents(const std::string& value)
{
  std::vector<std::string> toks = Tokenize(value,",");
  UTIL_THROW_IF2(toks.empty(), GetScoreProducerDescription()
                 << ": Empty tuneable-components");
  UTIL_THROW_IF2(toks.size()!=m_numScoreComponents, GetScoreProducerDescription()
                 << ": tuneable-components value has to be a comma-separated list of "
                 << m_numScoreComponents << " boolean values");

  m_tuneableComponents.resize(m_numScoreComponents);
  m_numTuneableComponents = m_numScoreComponents;

  for (size_t i = 0; i < toks.size(); ++i) {
    m_tuneableComponents[i] = Scan<bool>(toks[i]);
    if (!m_tuneableComponents[i]) {
      --m_numTuneableComponents;
    }
  }
}

void ExtractLex::Process(vector<string> &toksTarget, vector<string> &toksSource, vector<string> &toksAlign, size_t lineCount)
{
  std::vector<bool> m_sourceAligned(toksSource.size(), false)
                    , m_targetAligned(toksTarget.size(), false);

  vector<string>::const_iterator iterAlign;
  for (iterAlign = toksAlign.begin(); iterAlign != toksAlign.end(); ++iterAlign)
  {
    const string &alignTok = *iterAlign;
    
    vector<size_t> alignPos;
    Tokenize(alignPos, alignTok, "-");
    assert(alignPos.size() == 2);

	if (alignPos[0] >= toksSource.size())
	{
		cerr << "ERROR: alignment over source length. Alignment " << alignPos[0] << " at line " << lineCount << endl;
		continue;
	}
	if (alignPos[1] >= toksTarget.size())
	{
		cerr << "ERROR: alignment over target length. Alignment " << alignPos[1] << " at line " << lineCount << endl;
		continue;
	}
	
    assert(alignPos[0] < toksSource.size());
    assert(alignPos[1] < toksTarget.size());

    m_sourceAligned[ alignPos[0] ] = true;
    m_targetAligned[ alignPos[1] ] = true;

    const string &tmpSource = toksSource[ alignPos[0] ];
    const string &tmpTarget = toksTarget[ alignPos[1] ];
 
    const string *source = m_vocab.GetOrAdd(tmpSource);
    const string *target = m_vocab.GetOrAdd(tmpTarget);

    Process(target, source);
    
  }

  ProcessUnaligned(toksTarget, toksSource, m_sourceAligned, m_targetAligned);
}

Token *CharactersToTokens(Character text[], size_t nchars, size_t *ntokens_return)
{
    assert(text != NULL);

    puts("CharactersToTokens...");
    // nchars がトークン数の上限である。
    Token *res = GC_MALLOC(sizeof(Token) * nchars);
    size_t ntokens = 0;
    Character *p = text;

    while (p < text + nchars) {
	p = Tokenize(p, &res[ntokens++]);
    }
    puts("Done");
    res = GC_REALLOC(res, sizeof(Token) * ntokens);

    *ntokens_return = ntokens;
    return res;
}

bool cChannel::Parse(const std::string& data)
{
  std::vector<std::string> fields;

  Tokenize(data, fields, "|");

  if (fields.size() >= 4)
  {
    // Expected format:
    // ListTVChannels, ListRadioChannels
    // 0 = channel uid
    // 1 = channel external id/number
    // 2 = channel name
    // 3 = isencrypted ("0"/"1")
    // ListRadioChannels only: (TVServerXBMC >= v1.1.0.100)
    // 4 = iswebstream
    // 5 = webstream url
    // 6 = visibleinguide (TVServerXBMC >= v1.2.3.120)

    uid = atoi(fields[0].c_str());
    external_id = atoi(fields[1].c_str());
    name = fields[2];
    encrypted = (strncmp(fields[3].c_str(), "1", 1) == 0);
    
    if (fields.size() >= 6)
    {
      iswebstream = (strncmp(fields[4].c_str(), "1", 1) == 0);
      url = fields[5].c_str();

      if (fields.size() >= 7)
      {
        visibleinguide = (strncmp(fields[6].c_str(), "1", 1) == 0);
      }
    }

    return true;
  }
  else
  {
    return false;
  }
}

static ret_code GetNumber( char *string, int *pi, struct asm_tok tokenarray[] )
/*****************************************************************************/
{
    struct expr opndx;
    int i;
    int last;

    last = Tokenize( string, Token_Count+1, tokenarray, TOK_RESCAN );
    i = Token_Count+1;
    if( EvalOperand( &i, tokenarray, last, &opndx, EXPF_NOUNDEF ) == ERROR ) {
        return( ERROR );
    }
    /* v2.11: string constants are accepted ( although hardly useful ) */
    //if( opndx.kind != EXPR_CONST || opndx.quoted_string != NULL || tokenarray[i].token != T_FINAL ) {
    if( opndx.kind != EXPR_CONST || tokenarray[i].token != T_FINAL ) {
        return( EmitErr( SYNTAX_ERROR_EX, string ) );
    }
    *pi = opndx.value;
    return( NOT_ERROR );
}

	//bool stringCompare( const string &left, const string &right )
	//{
	//   for( string::const_iterator lit = left.begin(), rit = right.begin(); lit != left.end() && rit != right.end(); ++lit, ++rit )
	//      if( tolower( *lit ) < tolower( *rit ) )
	//         return true;
	//      else if( tolower( *lit ) > tolower( *rit ) )
	//         return false;
	//   if( left.size() < right.size() )
	//      return true;
	//   return false;
	//}
	StringVector CDirectoryManagerBase::GetDirectoriesFromString(const std::string& directoryListString)
	{
		StringVector tmp1 = Tokenize(directoryListString, "|");
		StringVector tmp2;
		for (StringVector::iterator it = tmp1.begin(); it != tmp1.end(); it++)
		{
			Trim(*it);
			if (!it->empty())
			{
				if (!IsPathEndOk(*it))
					*it += '\\';

				*it = SimplifyFilePath(*it);
				if (tmp2.Find(*it, false) == UNKNOWN_POS)
					tmp2.push_back(*it);
			}
		}

		return tmp2;
	}

nsresult
nsHtml5StringParser::ParseDocument(const nsAString& aSourceBuffer,
                                   nsIDocument* aTargetDoc,
                                   bool aScriptingEnabledForNoscriptParsing)
{
  MOZ_ASSERT(!aTargetDoc->GetFirstChild());

  NS_ENSURE_TRUE(aSourceBuffer.Length() <= INT32_MAX,
                 NS_ERROR_OUT_OF_MEMORY);

  mTreeBuilder->setFragmentContext(nullptr,
                                   kNameSpaceID_None,
                                   nullptr,
                                   false);

  mTreeBuilder->SetPreventScriptExecution(true);

  Tokenize(aSourceBuffer, aTargetDoc, aScriptingEnabledForNoscriptParsing);
  return NS_OK;
}

/*
** Append text to the tokenizer engine.
**
** This routine (actually the Tokenize() subroutine that is called
** by this routine) may invoke a callback procedure which could delete
** the HTML widget. 
*/
void HtmlTokenizerAppend(HtmlWidget *htmlPtr, const char *zText){
  int len = strlen(zText);
  if( htmlPtr->nText==0 ){
    htmlPtr->nAlloc = len + 100;
    htmlPtr->zText = HtmlAlloc( htmlPtr->nAlloc );
    TestPoint(0);
  }else if( htmlPtr->nText + len >= htmlPtr->nAlloc ){
    htmlPtr->nAlloc += len + 100;
    htmlPtr->zText = HtmlRealloc( htmlPtr->zText, htmlPtr->nAlloc );
    TestPoint(0);
  }
  if( htmlPtr->zText==0 ){
    htmlPtr->nText = 0;
    UNTESTED;
    return;
  }
  strcpy(&htmlPtr->zText[htmlPtr->nText], zText);
  htmlPtr->nText += len;
  htmlPtr->nComplete = Tokenize(htmlPtr);
}

int Encode (CFileReader &p_coFileReader)
{
	int iRetVal = 0;
	XML_Parser psoParser;

	psoParser = XML_ParserCreate ("UTF-8");
	if (NULL == psoParser) {
		iRetVal = ENOMEM;
		return iRetVal;
	}

	SDoc soDoc;

	/* регистрация обработчика данных */
	XML_SetElementHandler (psoParser, StartElementHandler, EndElementHandler);
	XML_SetXmlDeclHandler (psoParser, XmlDeclHandler);
	XML_SetDoctypeDeclHandler (psoParser, StartDoctypeDeclHandler, EndDoctypeDeclHandler);
	XML_SetUserData (psoParser, &soDoc);

	/* парсинг данных */
	char mcBuf[256];
	int iDataLen;
	int iIsFinal = 0;
	do {
		iDataLen = sizeof (mcBuf);
		if (p_coFileReader.ReadData ((unsigned char*)mcBuf, iDataLen))
			iIsFinal = 1;
		XML_Parse (psoParser, mcBuf, iDataLen, iIsFinal);
		if (iIsFinal)
			break;
	} while (1);

	Tokenize (soDoc);

	if (psoParser) {
		XML_ParserFree (psoParser);
		psoParser = NULL;
	}

	return iRetVal;
}