C++ StringTokenizer::tokenize方法代码示例

  void runTest()
  {
    StringTokenizer uut;

    Settings s;
    s.set(ConfigOptions::getTokenKeepNonWordsKey(), false);
    s.set(ConfigOptions::getTokenMinSizeKey(), 2);
    uut.setConfiguration(s);

    HOOT_STR_EQUALS("[2]{Hello, World.}", uut.tokenize("Hello World."));
    HOOT_STR_EQUALS("[1]{Dime}", uut.tokenize("5 & Dime"));
    HOOT_STR_EQUALS("[2]{Foo, Bar}", uut.tokenize("Foo ---- Bar"));

    s.set(ConfigOptions::getTokenKeepNonWordsKey(), true);
    s.set(ConfigOptions::getTokenMinSizeKey(), 1);
    uut.setConfiguration(s);

    HOOT_STR_EQUALS("[2]{Hello, World.}", uut.tokenize("Hello World."));
    HOOT_STR_EQUALS("[3]{5, &, Dime}", uut.tokenize("5 & Dime"));
    HOOT_STR_EQUALS("[3]{Foo, ----, Bar}", uut.tokenize("Foo ---- Bar"));

    s.set(ConfigOptions::getTokenKeepNonWordsKey(), true);
    s.set(ConfigOptions::getTokenMinSizeKey(), 3);
    uut.setConfiguration(s);

    HOOT_STR_EQUALS("[2]{Hello, World.}", uut.tokenize("Hello World."));
    HOOT_STR_EQUALS("[1]{Dime}", uut.tokenize("5 & Dime"));
    HOOT_STR_EQUALS("[3]{Foo, ----, Bar}", uut.tokenize("Foo ---- Bar"));
  }

bool SampleService::getSampleRange(Media& media)
{
   bool rval = false;

   // FIXME: change cache to just store sample range array? not entire media?
   // keep in mind that the code currently uses other information from
   // the media like the performer and title for the sample id3v2 tags

   // check with cache
   bool found = false;
   mSampleRangeCacheLock.lock();
   {
      if(mSampleRangeCache["cache"]->hasMember(media["id"]->getString()))
      {
         media = mSampleRangeCache["cache"][media["id"]->getString()];
         rval = found = true;
      }
   }
   mSampleRangeCacheLock.unlock();

   if(!found)
   {
      // get sample range for media from bitmunk
      Url url;
      url.format("/api/3.0/media/%" PRIu64, BM_MEDIA_ID(media["id"]));

      StringTokenizer st;
      if((rval = mNode->getMessenger()->getFromBitmunk(&url, media)))
      {
         // validate sample range
         st.tokenize(media["sampleRange"]->getString(), '-');
         if((rval = (st.getTokenCount() == 2)))
         {
            // set sample range start and end
            media["sampleRange"][0] =
               (uint32_t)strtoul(st.nextToken(), NULL, 10);
            media["sampleRange"][1] =
               (uint32_t)strtoul(st.nextToken(), NULL, 10);

            // update media length to use sample length
            int sampleLength =
               media["sampleRange"][1]->getUInt32() -
               media["sampleRange"][0]->getUInt32();
            media["length"] = sampleLength;

            // update media title to include "- Bitmunk Sample"
            string title = media["title"]->getString();
            title.append(" - Bitmunk Sample");
            media["title"] = title.c_str();

            // update cache
            mSampleRangeCacheLock.lock();
            {
               if(mSampleRangeCache["cache"]->length() + 1 >=
                  mSampleRangeCache["capacity"]->getInt32())
               {
                  // clear cache, capacity reached
                  mSampleRangeCache->clear();
               }

               mSampleRangeCache["cache"][media["id"]->getString()] = media;
            }
            mSampleRangeCacheLock.unlock();
         }
         else
         {
            // no sample range available
            media["sampleRange"][0] = 0;
            media["sampleRange"][1] = 0;
         }

         // clear any sample content-length
         media->removeMember("sampleContentLength");
      }
   }

   return rval;
}

ShaderComp::StageMask
ShaderFactory::createMains(const ShaderComp::FunctionLocationMap&    functions,
                           const VirtualProgram::ShaderMap&          in_shaders,
                           std::vector< osg::ref_ptr<osg::Shader> >& out_shaders) const
{
    StageMask stages =
        ShaderComp::STAGE_VERTEX |
        ShaderComp::STAGE_FRAGMENT;

    FunctionLocationMap::const_iterator f;

    // collect the "model" stage vertex functions:
    f = functions.find( LOCATION_VERTEX_MODEL );
    const OrderedFunctionMap* modelStage = f != functions.end() ? &f->second : 0L;

    // collect the "view" stage vertex functions:
    f = functions.find( LOCATION_VERTEX_VIEW );
    const OrderedFunctionMap* viewStage = f != functions.end() ? &f->second : 0L;

    // geometry shader functions:
    f = functions.find( LOCATION_TESS_CONTROL );
    const OrderedFunctionMap* tessControlStage = f != functions.end() ? &f->second : 0L;

    // geometry shader functions:
    f = functions.find( LOCATION_TESS_EVALUATION );
    const OrderedFunctionMap* tessEvalStage = f != functions.end() ? &f->second : 0L;

    // geometry shader functions:
    f = functions.find( LOCATION_GEOMETRY );
    const OrderedFunctionMap* geomStage = f != functions.end() ? &f->second : 0L;

    // collect the "clip" stage functions:
    f = functions.find( LOCATION_VERTEX_CLIP );
    const OrderedFunctionMap* clipStage = f != functions.end() ? &f->second : 0L;

    // fragment shader coloring functions:
    f = functions.find( LOCATION_FRAGMENT_COLORING );
    const OrderedFunctionMap* coloringStage = f != functions.end() ? &f->second : 0L;

    // fragment shader lighting functions:
    f = functions.find( LOCATION_FRAGMENT_LIGHTING );
    const OrderedFunctionMap* lightingStage = f != functions.end() ? &f->second : 0L;

    // fragment shader lighting functions:
    f = functions.find( LOCATION_FRAGMENT_OUTPUT );
    const OrderedFunctionMap* outputStage = f != functions.end() ? &f->second : 0L;

    // what do we need to build?
    bool hasGS  = geomStage        && !geomStage->empty();
    bool hasTCS = tessControlStage && !tessControlStage->empty();
    bool hasTES = tessEvalStage    && !tessEvalStage->empty();
    bool hasFS  = true;
    bool hasVS  = true;
    
    // where to insert the view/clip stage vertex functions:
    bool viewStageInGS  = hasGS;
    bool viewStageInTES = !viewStageInGS && hasTES;
    bool viewStageInVS  = !viewStageInTES && !viewStageInGS;
    
    bool clipStageInGS  = hasGS;
    bool clipStageInTES = hasTES && !hasGS;
    bool clipStageInVS  = !clipStageInGS && !clipStageInTES;

    // search for pragma varyings and build up our interface block definitions.
    typedef std::set<std::string> VarDefs;
    VarDefs varDefs;

    // built-ins:
    varDefs.insert( "vec4 vp_Color" );
    varDefs.insert( "vec3 vp_Normal" );
    varDefs.insert( "vec4 vp_Vertex" );

    // parse the vp_varyings (which were injected by the ShaderLoader)
    for(VirtualProgram::ShaderMap::const_iterator s = in_shaders.begin(); s != in_shaders.end(); ++s )
    {
        osg::Shader* shader = s->data()._shader->getNominalShader();
        if ( shader )
        {
            ShaderLoader::getAllPragmaValues(shader->getShaderSource(), "vp_varying", varDefs);
        }
    }

    Variables vars;
    for(VarDefs::iterator i = varDefs.begin(); i != varDefs.end(); ++i) 
    {
        std::vector<std::string> tokens;        
        StringTokenizer st;
        st.addDelims( " \t", false );
        st.addDelims( "[]", true );
        st.tokenize( *i, tokens ); //(*i, tokens, " \t", "", false, true);
        if ( tokens.size() >= 2 )
        {
            int p=0;
            Variable v;
            if ( tokens[p] == "flat" || tokens[p] == "nonperspective" || tokens[p] == "smooth" )
            {
                v.interp = tokens[p++];
            }

            if ( p+1 < tokens.size() )
//.........这里部分代码省略.........

//...................................................... tokenize ...
bool FileTokenizer::tokenize() {
    ifstream currentFile;             //file to be tokenized
    string inputLine;                 //line that will be read from the file
    StringTokenizer * tkn;         //String tokenizer
    int i;                            //index variable
    string token;                     //token currently extracted
    bool res = true;
    
    tkn = new StringTokenizer();
    tkn->setSeparators(currentSeparators);
    
	// [1] Initialize position and open the file to be tokenized
	tokenPos = 0;
	tokens.clear();
	currentFile.open(currentFileName.c_str(), std::ifstream::in);

	if (currentFile.fail()) {
		std::cerr << "Can't open file [" << currentFileName << "]" << std::endl;

		res = false;

	} else {
        // [2] Read input while the file has not been finished, and tokenize
        while (currentFile.good()) {
            //Get next line
            std::getline(currentFile, inputLine);

			#ifndef _WIN32
            inputLine = StringUtilities::removeAtEnd(inputLine, __CRLF);
			#endif

            //Tokenize
            tkn->setLine(inputLine);
            tkn->tokenize(true);

            /*
             * Add tokens to the token buffer.
             *
             * Tokens are added while they are "correct" tokens. A correct token is a
             * token that is not empty, does not contain "#", and does not appear
             * after a "#" character in the line.
             */
            i = 0;
            while (i < tkn->getNumTokens()) {
                token = tkn->getToken(i);

                if (token == "") {
                    //The token is blank. Do not add but check next token
                    i = i + 1;
                } else {
                    //The token is not blank, and has no comments inside it
                    tokens.push_back(token);
                    i = i + 1;
                }
            }
        }
    }

    currentFile.close();
    
    //Free memory of the ATMStringTokenizer object
    delete tkn;

    return res;
}

void CookieJar::readCookies(HttpHeader* header, CookieOrigin origin)
{
    // parse cookies if appropriate cookie field exists
    const char* field = (origin == Server ? "Set-Cookie" : "Cookie");
    int count = header->getFieldCount(field);
    if(count > 0)
    {
        // Set-Cookie: cookie1_name=cookie1_value; max-age=0; path=/
        // Set-Cookie: c2=v2; expires=Thu, 21-Aug-2008 23:47:25 GMT; path=/
        // Cookie: cookie1_name=cookie1_value; cookie2_name=cookie2_value

        // parse cookies by semi-colons (cannot parse by commas because
        // the "expires" value may contain a comma and no one follows the
        // standard)
        string cookies;
        Cookie cookie(NULL);
        TimeZone gmt = TimeZone::getTimeZone("GMT");
        Date d;
        Date now;
        bool name;
        StringTokenizer pairs;
        for(int i = 0; i < count; ++i)
        {
            name = true;
            header->getField(field, cookies, i);
            pairs.tokenize(cookies.c_str(), ';');
            while(pairs.hasNextToken())
            {
                // get next token (name=value)
                const char* token = pairs.nextToken();

                // get name part of token
                size_t nameLength = strcspn(token, "=");
                char tmpName[nameLength + 1];
                strncpy(tmpName, token, nameLength);
                tmpName[nameLength] = 0;

                // trim whitespace from name
                char* namePtr = (tmpName + nameLength);
                for(; namePtr != tmpName && *namePtr == ' '; --namePtr)
                {
                    *namePtr = 0;
                }
                for(namePtr = tmpName; *namePtr != 0 && *namePtr == ' ';
                        ++namePtr);

                // get value part of token (ensure value length is at least 0 in
                // case bad parsing because of no equals sign after name)
                size_t valueLength = strlen(token);
                valueLength = (nameLength == valueLength ?
                               0 : valueLength - nameLength - 1);
                char tmpValue[valueLength + 1];
                if(valueLength > 0)
                {
                    strncpy(tmpValue, token + nameLength + 1, valueLength);
                }
                tmpValue[valueLength] = 0;

                if(origin == Client)
                {
                    // set cookie
                    setCookie(namePtr, tmpValue, 0, false, false);
                }
                else
                {
                    if(name)
                    {
                        // first token *must* be cookie name = cookie value
                        cookie = Cookie();
                        cookie["name"] = namePtr;
                        cookie["value"] = tmpValue;
                        name = false;
                    }
                    else
                    {
                        if(strcmp(namePtr, "expires") == 0)
                        {
                            // parse expiration time
                            if(d.parse(tmpValue, HttpHeader::sDateFormat, &gmt))
                            {
                                int64_t age = d.getSeconds() - now.getSeconds();
                                cookie["maxAge"] = (age <= 0 ? 0 : age);
                            }
                            else
                            {
                                // bad date format, use maxAge of 0
                                cookie["maxAge"] = 0;
                            }
                        }
                        else if(strcasecmp(namePtr, "secure") == 0)
                        {
                            // cookie is secure
                            cookie["secure"] = true;
                        }
                        else if(strcasecmp(namePtr, "HttpOnly") == 0)
                        {
                            // cookie is http-only (non-javascript)
                            cookie["httpOnly"] = true;
                        }
                        else
//.........这里部分代码省略.........