C++ TStr::GetLc方法代码示例

TEST(TStr, GetLc) {
	TStr Mixedcase = "AbCd";
	TStr Lowercase = "abcd";	
	TStr Empty = "";
	EXPECT_EQ(Lowercase, Mixedcase.GetLc());
	EXPECT_EQ(Empty, Empty.GetLc());
}

int TTmInfo::GetDayOfWeekN(const TStr& DayOfWeekNm, const TLoc& Loc){
  EnsureInit();
  int DayOfWeekN=-1;
  switch (Loc){
    case lUs: DayOfWeekN=UsDayOfWeekNmV.SearchForw(DayOfWeekNm.GetLc()); break;
    case lSi: DayOfWeekN=SiDayOfWeekNmV.SearchForw(DayOfWeekNm.GetLc()); break;
    default: Fail;
  }
  if (DayOfWeekN==-1){return -1;} else {return DayOfWeekN+1;}
}

int TTmInfo::GetMonthN(const TStr& MonthNm, const TLoc& Loc){
  EnsureInit();
  int MonthN=-1;
  switch (Loc){
    case lUs: MonthN=UsMonthNmV.SearchForw(MonthNm.GetLc()); break;
    case lSi: MonthN=SiMonthNmV.SearchForw(MonthNm.GetLc()); break;
    default: Fail;
  }
  if (MonthN==-1){return -1;} else {return MonthN+1;}
}

bool GetBoolArg(const PXmlTok& QueryXml, const TStr& ArgNm, const bool DfVal)
{
	if (QueryXml.Empty()) return DfVal;
	TStr val = QueryXml->GetArgVal(ArgNm, "");
	if (val == "") return DfVal;
	val = val.GetLc();
	if (val == "true" || val == "1")
		return true;
	return false;
}

void TTokenizerHtml::GetTokens(const PSIn& SIn, TStrV& TokenV) const {
	THtmlLx HtmlLx(SIn, false);
    // traverse html string symbols
	while (HtmlLx.Sym!=hsyEof){
		if (HtmlLx.Sym==hsyStr){
			TStr UcStr = HtmlLx.UcChA;
			// check if stop word
			if ((SwSet.Empty()) || (!SwSet->IsIn(UcStr))) {
				TStr TokenStr = ToUcP ? UcStr : TStr(HtmlLx.ChA);
				if (!Stemmer.Empty()) { 
					TokenStr = Stemmer->GetStem(TokenStr); }
				TokenV.Add(TokenStr.GetLc());
			}
		}
		// get next symbol
		HtmlLx.GetSym();
	}

 //   // tokenize
 //   TStrV TokenStrV; Tokenizer->GetTokens(TextStr, TokenStrV);
 //   // transform words to IDs
 //   const int Tokens = TokenStrV.Len();
 //   TIntV TokenIdV(Tokens, 0);
 //   for (int TokenN = 0; TokenN < Tokens; TokenN++) {
 //       // add token to the hashtable of all tokens
 //       const int TokenId = WordH.AddKey(TokenStrV[TokenN].GetUc());
 //       // keep track of it's count
 //       WordH[TokenId]++;
 //       // and prepare a token vector for ngram base
 //       TokenIdV.Add(TokenId);
 //   }
	//// extract the n-grams
 //   TNGramDescV NGramDescV;
	//NGramBs->AddDocTokIdV(TokenIdV, StoreThreshold, NGramDescV);
 //   // get string representations of n-grams above threshold
 //   TStrH NGramH;
 //   for (int NGramDescN = 0; NGramDescN < NGramDescV.Len(); NGramDescN++) {
 //       const TNGramDesc& NGramDesc = NGramDescV[NGramDescN];
 //       // make it into a string
 //       const TIntV& NGramTokenIdV = NGramDesc.TokIdV;
 //       TChA NGramChA = WordH.GetKey(NGramTokenIdV[0]);
 //       for (int NGramTokenIdN = 1; NGramTokenIdN < NGramTokenIdV.Len(); NGramTokenIdN++) {
 //           NGramChA += ' '; NGramChA += WordH.GetKey(NGramTokenIdV[NGramTokenIdN]);
 //       }
 //       // remember the ngram, if not stopword
 //       if (!SwSet->IsIn(NGramChA)) { NGramH.AddDat(NGramChA); }
 //   }
 //   // remember n-grams above threshold
 //   int NGramKeyId = NGramH.FFirstKeyId();
 //   while (NGramH.FNextKeyId(NGramKeyId)) {
 //       const TStr& NGramStr = NGramH.GetKey(NGramKeyId);
 //       // add to the result list
 //       ConceptV.Add(TOgNewsConcept(NGramStr, EmtpyStr));
 //   }
}

TStemmerType TStemmer::GetStemmerType(const TStr& StemmerTypeNm) {
    TStr LcStemmerTypeNm=StemmerTypeNm.GetLc();
    if (LcStemmerTypeNm=="none") {
        return stmtNone;
    }
    else if (LcStemmerTypeNm=="porter") {
        return stmtPorter;
    }
    else {
        return stmtNone;
    }
}

int TRSet::GetHitN(const TStr& UrlStr, const bool& LcP) const {
  int Hits=GetHits();
  TStr LcUrlStr=UrlStr.GetLc();
  for (int HitN=0; HitN<Hits; HitN++){
    if (LcP){
      if (GetHitUrlStr(HitN).GetLc()==LcUrlStr){return HitN;}
    } else {
      if (GetHitUrlStr(HitN)==UrlStr){return HitN;}
    }
  }
  return -1;
}

TSwSetType TSwSet::GetSwSetType(const TStr& SwSetTypeNm){
  TStr LcSwSetTypeNm=SwSetTypeNm.GetLc();
  if (LcSwSetTypeNm=="none"){return swstNone;}
  else if (LcSwSetTypeNm=="en8"){return swstEn8;}
  else if (LcSwSetTypeNm=="en425"){return swstEn425;}
  else if (LcSwSetTypeNm=="en523"){return swstEn523;}
  else if (LcSwSetTypeNm=="enMsdn"){return swstEnMsdn;}
  else if (LcSwSetTypeNm=="es"){return swstEs;}
  else if (LcSwSetTypeNm=="german"){return swstGe;}
  else if (LcSwSetTypeNm=="siyuascii"){return swstSiYuAscii;}
  else if (LcSwSetTypeNm=="siisoce"){return swstSiIsoCe;}
  else {return swstNone;}
}

void THtml::GetTokens(const PSIn& SIn, TStrV& TokenV) const {
	THtmlLx HtmlLx(SIn, false);
    // traverse html string symbols
	while (HtmlLx.Sym!=hsyEof){
		if (HtmlLx.Sym==hsyStr){
			TStr UcStr = HtmlLx.UcChA;
			// check if stop word
			if ((SwSet.Empty()) || (!SwSet->IsIn(UcStr))) {
				TStr TokenStr = ToUcP ? UcStr : TStr(HtmlLx.ChA);
				if (!Stemmer.Empty()) { 
					TokenStr = Stemmer->GetStem(TokenStr); }
				TokenV.Add(TokenStr.GetLc());
			}
		}
		// get next symbol
		HtmlLx.GetSym();
	}
}

/////////////////////////////////////////////////
// BagOfWords-Files
void TBowFl::LoadHtmlTxt(
 PBowDocBs BowDocBs, const TStr& FPath, TIntV& NewDIdV,
 const bool& RecurseDirP, const int& MxDocs,
 const bool& SaveDocP, const PNotify& Notify) {
  // prepare file-directory traversal
  TStr LcNrFPath=TStr::GetNrFPath(FPath).GetLc();
  Notify->OnStatus("Creating Bow from file-path " + FPath + " ...");
  TFFile FFile(FPath, "", RecurseDirP);
  // traverse files
  TStr FNm; int Docs=0; NewDIdV.Clr();
  while (FFile.Next(FNm)){
    Docs++; if ((MxDocs!=-1)&&(Docs>MxDocs)){break;}
    Notify->OnStatus(TStr::Fmt("%d\r", Docs));
    // prepare document-name
    if (TFile::Exists(FNm)) { //B:
        TStr DocNm=FNm.GetLc();
        if (DocNm.IsPrefix(LcNrFPath)){
          DocNm=DocNm.GetSubStr(LcNrFPath.Len(), DocNm.Len()-1);}
        // categories
        TStrV CatNmV; TStr CatNm;
        if (DocNm.IsChIn('/')){
          TStr Str; DocNm.SplitOnCh(CatNm, '/', Str);
        } else if (DocNm.IsChIn('\\')){
          TStr Str; DocNm.SplitOnCh(CatNm, '\\', Str);
        }
        if (!CatNm.Empty()){
          CatNmV.Add(CatNm);}
        // load document-content
        TStr DocStr=TStr::LoadTxt(FNm);
        // add document to bow
        NewDIdV.Add(BowDocBs->AddHtmlDoc(DocNm, CatNmV, DocStr, SaveDocP));
    }
  }
  Notify->OnStatus(TStr::Fmt("%d", Docs));
  // return results
  Notify->OnStatus("Done.");
  BowDocBs->AssertOk();
}

void TGgSchBs::SetAuthCrawled(const TStr& AuthNm){
  AuthNmToCrawlPH.AddDat(AuthNm.GetLc())=true;
}

void BigMain(int argc, char* argv[]) {
  TExeTm ExeTm;
  Env = TEnv(argc, argv, TNotify::StdNotify);
  Env.PrepArgs("QuotesApp");
  const TStr ToDo = Env.GetIfArgPrefixStr("-do:", "", "To do").GetLc();
  if (Env.IsEndOfRun()) {
    printf("To do:\n");
    printf("    MkDataset         : Make memes dataset (extract quotes and save txt)\n");
    printf("    ExtractSubset     : Extract a subset of memes containing particular words\n");
    printf("    MemesToQtBs       : Load memes dataset and create quote base\n");
    printf("    MkClustNet        : Build cluster network from the quote base\n");
    return;
  }	
#pragma region mkdataset
  // extract quotes and links and make them into a single file
  if (ToDo == "mkdataset") {
    const TStr InFNm = Env.GetIfArgPrefixStr("-i:", "files.txt", "Spinn3r input files (one file per line)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "Spinn3r-dataset.txt", "Output file");
    const int MinQtWrdLen = Env.GetIfArgPrefixInt("-w:", 3, "Minimum quote word length");
    const TStr UrlFNm = Env.GetIfArgPrefixStr("-u:", "", "Seen url set (THashSet<TMd5Sig>) file name");
    const bool UrlOnlyOnce = Env.GetIfArgPrefixBool("-q:", true, "Only keep unique Urls");
    //// parse directly from Spinn3r
    TStr Spinn3rFNm;
    THashSet<TMd5Sig> SeenUrlSet;
    if (UrlOnlyOnce && ! UrlFNm.Empty()) {  // keep track of already seen urls (so that there are no duplicate urls)
      TFIn FIn(UrlFNm);  SeenUrlSet.Load(FIn);
    }
    FILE *F = fopen(OutFNm.CStr(), "wt");
    TFIn FIn(InFNm);
    int Items=0;
    for (int f=0; FIn.GetNextLn(Spinn3rFNm); f++) {
      TQuoteExtractor QE(Spinn3rFNm.ToTrunc());
      printf("Processing %02d: %s [%s]\n", f+1, Spinn3rFNm.CStr(), TExeTm::GetCurTm());
      fflush(stdout);
      for (int item = 0; QE.Next(); item++) {
        const TMd5Sig PostMd5(QE.PostUrlStr);
        if (QE.QuoteV.Empty() && QE.LinkV.Empty()) { continue; } // no quotes, no links
        if (UrlOnlyOnce) {
          if (SeenUrlSet.IsKey(PostMd5)) { continue; }
          SeenUrlSet.AddKey(PostMd5);
        }
        fprintf(F, "P\t%s\n", QE.PostUrlStr.CStr());
        //if (QE.PubTm > TSecTm(2008,8,30) || QE.PubTm < TSecTm(2008,7,25)) { printf("%s\n", QE.PubTm.GetStr().CStr()); }
        fprintf(F, "T\t%s\n", QE.PubTm.GetYmdTmStr().CStr());
        for (int q = 0; q < QE.QuoteV.Len(); q++) {
          if (TStrUtil::CountWords(QE.QuoteV[q]) >= MinQtWrdLen) {
            fprintf(F, "Q\t%s\n", QE.QuoteV[q].CStr()); }
        }
        for (int l = 0; l < QE.LinkV.Len(); l++) {
          fprintf(F, "L\t%s\n", QE.LinkV[l].CStr()); }
        fprintf(F, "\n");
        if (item>0 && item % Kilo(100) == 0) {
          QE.DumpStat();  QE.ExeTm.Tick(); }
        Items++;
      }
      printf("file done. Total %d all posts, %d all items\n", SeenUrlSet.Len(), Items);
      fflush(stdout);
    }
    printf("all done. Saving %d post urls\n", SeenUrlSet.Len());  fflush(stdout);
    if (! SeenUrlSet.Empty()) {
      TFOut FOut(OutFNm.GetFMid()+".SeenUrlSet");
      SeenUrlSet.Save(FOut);
    }
    fclose(F);
  }
#pragma endregion mkdataset

#pragma region extractsubset
  // save posts with memes containing particular words
  else if (ToDo == "extractsubset") {
    const TStr InFNmWc = Env.GetIfArgPrefixStr("-i:", "memes_*.rar", "Input file prefix");
    const bool IsInFNmWc = Env.GetIfArgPrefixBool("-w:", true, "Input is wildcard (else a file with list of input files)");
    const TStr OutFNm = Env.GetIfArgPrefixStr("-o:", "memes-subset.txt", "Output memes file");
    const TStr WordsFNm = Env.GetIfArgPrefixStr("-p:", "phrases-in.txt", "Phrases that memes have to contain");

    TChAV CatchMemeV;// = TStr::GetV("great depression", "economic meltdown", "recession had bottomed out", "green shoots", "slow recovery", "gradual recovery");
    printf("Loading %s\n", WordsFNm.CStr());
    { TFIn FIn(WordsFNm);
    for (TStr Ln; FIn.GetNextLn(Ln); ) {
      printf("  %s\n", Ln.GetLc().CStr());
      CatchMemeV.Add(Ln.GetLc()); }
    }
    printf("%d strings loaded\n", CatchMemeV.Len());
    TFOut FOut(OutFNm);
    TMemesDataLoader Memes(InFNmWc, IsInFNmWc);
    for (int posts = 0, nsave=0; Memes.LoadNext(); posts++) {
      bool DoSave = false;
      for (int m = 0; m < Memes.MemeV.Len(); m++) {
        for (int i = 0; i < CatchMemeV.Len(); i++) {
          if (Memes.MemeV[m].SearchStr(CatchMemeV[i]) != -1) {
            DoSave=true; break; }
        }
        if (DoSave) { break; }
      }
      if (DoSave) { Memes.SaveTxt(FOut); nsave++; }
      if (posts % Mega(1) == 0) {
        printf("%dm posts, %d saved\n", posts/Mega(1), nsave);
        FOut.Flush();
      }
    }
//.........这里部分代码省略.........