C++ TChA::Empty方法代码示例

PNGramBs TNGramBs::GetNGramBsFromLnDoc(
 const TStr& LnDocFNm, const bool& NamedP, const int& MxDocs,
 const int& MxNGramLen, const int& MnNGramFq,
 const PSwSet& SwSet, const PStemmer& Stemmer){
  // create n-gram-base
  /* printf("Generating frequent n-grams (MaxLen:%d MinFq:%d) ...\n", MxNGramLen, MnNGramFq); */
  PNGramBs NGramBs=TNGramBs::New(MxNGramLen, MnNGramFq, SwSet, Stemmer);
  // interations over document-set
  while (!NGramBs->IsFinished()){
    // open line-doc file
    TFIn FIn(LnDocFNm); char Ch=' '; int Docs=0;
    while (!FIn.Eof()){
	  if(Ch == '\r' || Ch == '\n'){Ch = FIn.GetCh(); continue;}
      Docs++; if ((MxDocs!=-1)&&(Docs>=MxDocs)){break;}
      //printf("%d\r", Docs);
      // document name
      TChA DocNm;
      if (NamedP){
        Ch=FIn.GetCh();
        while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')&&(Ch!=' ')){
          DocNm+=Ch; Ch=FIn.GetCh();}
        DocNm.Trunc();
        if (DocNm.Empty()){Docs--; continue;}
      }
      // categories
      TStrV CatNmV;
      forever {
        while ((!FIn.Eof())&&(Ch==' ')){Ch=FIn.GetCh();}
        if (Ch=='!'){
          TChA CatNm;
          while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')&&(Ch!=' ')){
            CatNm+=Ch; Ch=FIn.GetCh();}
          if (!CatNm.Empty()){CatNmV.Add(CatNm);}
        } else {
          break;
        }
      }
      // document text
      TChA DocChA;
      while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')){
        DocChA+=Ch; Ch=FIn.GetCh();}
      // extract words & update ngram-base
      /* printf("  Pass %2d: %6d Docs\r", NGramBs->GetPassN(), Docs); */
      _UpdateNGramBsFromHtmlStr(NGramBs, DocChA, SwSet, Stemmer);
    }
    NGramBs->ConcPass();
  }
  /* printf("\nDone.\n"); */
  // return
  return NGramBs;
}

void TCpDoc::SaveLnDocToCpd(
 const TStr& LnDocFNm, const TStr& OutCpdFNm, const bool& NamedP, const int& MxDocs){
  printf("Saving Line-Document '%s' to '%s' ...\n", LnDocFNm.CStr(), OutCpdFNm.CStr());
  // create output file
  PSOut SOut=TFOut::New(OutCpdFNm);
  // open line-doc file
  TFIn FIn(LnDocFNm); char Ch=' '; int Docs=0;
  while (!FIn.Eof()){
    Docs++; if ((MxDocs!=-1)&&(Docs>=MxDocs)){break;}
    printf("%d\r", Docs);
    // document name
    TChA DocNm;
    if (NamedP){
      Ch=FIn.GetCh();
      while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')&&(Ch!=' ')){
        DocNm+=Ch; Ch=FIn.GetCh();}
      DocNm.Trunc();
      if (DocNm.Empty()){Docs--; continue;}
    }
    // categories
    TStrV CatNmV;
    forever {
      while ((!FIn.Eof())&&(Ch==' ')){Ch=FIn.GetCh();}
      if (Ch=='!'){
        if (!FIn.Eof()){Ch=FIn.GetCh();}
        TChA CatNm;
        while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')&&(Ch!=' ')){
          CatNm+=Ch; Ch=FIn.GetCh();}
        if (!CatNm.Empty()){CatNmV.Add(CatNm);}
      } else {
        break;
      }
    }
    // document text
    TChA DocChA;
    while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')){
      DocChA+=Ch; Ch=FIn.GetCh();}
    // skip empty documents (empty lines)
    if (DocNm.Empty()&&DocChA.Empty()){
      continue;}
    // create & save cpd document
    PCpDoc CpDoc=TCpDoc::New();
    CpDoc->DocNm=DocNm;
    CpDoc->ParStrV.Add(DocChA, 1);
    for (int CatNmN=0; CatNmN<CatNmV.Len(); CatNmN++){
      CpDoc->TopCdNmV.Add(CatNmV[CatNmN]);}
    CpDoc->Save(*SOut);
  }
  printf("\nDone.\n");
}

// remove ending /, /index.html, etc. and strip starting www.
bool TStrUtil::GetNormalizedUrl(const TChA& UrlIn, const TChA& BaseUrl, TChA& UrlOut) {
    UrlOut = UrlIn;
    if (StripEnd(UrlIn, "/", UrlOut)) {}
    else if (StripEnd(UrlIn, "/index.html", UrlOut)) {}
    else if (StripEnd(UrlIn, "/index.htm", UrlOut)) {}
    else if (StripEnd(UrlIn, "/index.php", UrlOut)) {}
    if (! (UrlOut.IsPrefix("http://") || UrlOut.IsPrefix("ftp://"))) {
        // if UrlIn is relative url, try combine it with BaseUrl
        if (UrlIn.Empty() || ! (BaseUrl.IsPrefix("http://") || BaseUrl.IsPrefix("ftp://"))) {
            //printf("** Bad URL: base:'%s' url:'%s'\n", BaseUrl.CStr(), UrlIn.CStr());
            return false;
        }
        TChA Out;
        if (! GetNormalizedUrl(BaseUrl, TChA(), Out)) {
            return false;
        }
        if (UrlIn[0] != '/') {
            Out.AddCh('/');
        }
        Out += UrlOut;
        UrlOut = Out;
    }
    // http://www. --> http://
    if (UrlOut.IsPrefix("http://www.")) {
        TStr prefix("http://");
        UrlOut = prefix + UrlOut.GetSubStr(11, TInt::Mx);
    }
    UrlOut.ToLc();
    return true;
}

bool TFIn::GetNextLnBf(TChA& LnChA) {
  int Status;
  int BfN;        // new pointer to the end of line
  int BfP;        // previous pointer to the line start
  bool CrEnd;     // last character in previous buffer was CR

  LnChA.Clr();

  CrEnd = false;
  do {
    if (BfC >= BfL) {
      // reset the current pointer, FindEol() will read a new buffer
      BfP = 0;
    } else {
      BfP = BfC;
    }
    Status = FindEol(BfN,CrEnd);
    if (Status >= 0) {
      if (BfN-BfP > 0) {
        LnChA.AddBf(&Bf[BfP],BfN-BfP);
      }
      if (Status == 1) {
        // got a complete line
        return true;
      }
    }
    // get more data, if the line is incomplete
  } while (Status == 0);

  // eof or the last line has no newline
  return !LnChA.Empty();
}

TStr TStrUtil::GetStr(const TUInt64V& UInt64V, const TStr& DelimiterStr) {
	TChA ResChA;
	for (int N = 0; N < UInt64V.Len(); N++) {
		if (!ResChA.Empty()) { ResChA += DelimiterStr; }
		ResChA += UInt64V[N].GetStr();
	}
	return ResChA;
}

void TBowFl::LoadLnDocTxt(PBowDocBs BowDocBs, const TStr& LnDocFNm,
 TIntV& NewDIdV, const bool& NamedP, const int& MxDocs, const bool& SaveDocP) {
  // open line-doc file
  NewDIdV.Clr(); TFIn FIn(LnDocFNm); char Ch=' '; int Docs=0;
  while (!FIn.Eof()){
    Docs++; if ((MxDocs!=-1)&&(Docs>=MxDocs)){break;}
    printf("%d\r", Docs);
    // document name
    TChA DocNm;
    Ch=FIn.GetCh();
    if (NamedP){
      while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')&&(Ch!=' ')){
        DocNm+=Ch; Ch=FIn.GetCh();}
      DocNm.Trunc();
      if (DocNm.Empty()){Docs--; continue;}
    } else {
        DocNm = TInt::GetStr(Docs);
    }
    // categories
    TStrV CatNmV;
    forever {
      while ((!FIn.Eof())&&(Ch==' ')){Ch=FIn.GetCh();}
      if (Ch=='!'){
        if (!FIn.Eof()){Ch=FIn.GetCh();}
        TChA CatNm;
        while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')&&(Ch!=' ')){
          CatNm+=Ch; Ch=FIn.GetCh();}
        if (!CatNm.Empty()){CatNmV.Add(CatNm);}
      } else {
        break;
      }
    }
    // document text
    TChA DocChA;
    while ((!FIn.Eof())&&(Ch!='\r')&&(Ch!='\n')){
      DocChA+=Ch; Ch=FIn.GetCh();}
    // skip empty documents (empty lines)
    if (DocNm.Empty()&&DocChA.Empty()){
      continue;}
    // add document to document-base
    NewDIdV.Add(BowDocBs->AddHtmlDoc(DocNm, CatNmV, DocChA, SaveDocP));
  }
  // return document-base
  BowDocBs->AssertOk();
  printf("\n");
}

TStr THttpLx::GetUrlStr(){
  TChA UrlChA;
  while ((!Eof())&&(!ChDef.IsSp(Ch))){
    UrlChA+=Ch; GetCh();}
  if (UrlChA.Empty()){
    throw THttpEx(heUrlEmpty);}
  return UrlChA;
}

bool TSIn::GetNextLn(TChA& LnChA){
  LnChA.Clr();
  while (!Eof()){
    const char Ch=GetCh();
    if (Ch=='\n'){return true;}
    if (Ch=='\r' && PeekCh()=='\n'){GetCh(); return true;}
    LnChA.AddCh(Ch);
  }
  return !LnChA.Empty();
}

TStr TRulette::GetBidStr() const {
  TChA BidChA;
  for (int BidFldN=0; BidFldN<37; BidFldN++){
    if (FldBidV[BidFldN]!=0){
      if (!BidChA.Empty()){BidChA+=';';}
      BidChA+=TInt::GetStr(BidFldN); BidChA+=':';
      BidChA+=TInt::GetStr(FldBidV[BidFldN]);
    }
  }
  return BidChA;
}

PUrl THttpLx::GetUrl(){
  TChA UrlChA;
  while ((!Eof())&&(!ChDef.IsSp(Ch))){
    UrlChA+=Ch; GetCh();}
  if (UrlChA.Empty()){
    throw THttpEx(heUrlEmpty);}
  static TStr LocalBaseUrlStr="http://localhost/";
  PUrl Url=PUrl(new TUrl(UrlChA, LocalBaseUrlStr));
  if (!Url->IsOk()){
    throw THttpEx(heBadUrl);}
  return Url;
}

/////////////////////////////////////////////////
// Translation-Evaluation-Scores
void TEvalScore::Tokenize(const TStr& Str, TStrV& TokenV) {
    TStr LcStr = GetLc(Str);
    TokenV.Clr(); TChA WordChA; 
    const int Chs = LcStr.Len();
    for (int ChN = 0; ChN < Chs; ChN++) {
        const char Ch = LcStr[ChN];
        if (IsChar(Ch)) { 
            WordChA += Ch; 
        } else if (!WordChA.Empty()) {
            TokenV.Add(WordChA);
            WordChA.Clr();
        }
    }        
}

void MakeSlashdotSignNet(const TStr InFNm, TStr OutFNm, TStr Desc, THashSet<TChA> NIdSet) {
  //THashSet<TChA> NIdSet;
  TChA LnStr;
  TVec<char *> WrdV;
  int Sign;
  //PSignNet Net = TSignNet::New();
  TPt<TNodeEDatNet<TInt, TInt> >  Net = TNodeEDatNet<TInt, TInt>::New();
  int i = 0;
  for (TFIn FIn(InFNm); FIn.GetNextLn(LnStr); ) {
    if (LnStr.Empty() || LnStr[0]=='#') { continue; }
    LnStr.ToLc();
    TStrUtil::SplitOnCh(LnStr, WrdV, '\t', false);
    //NIdSet.AddKey(WrdV[0]);
    if (strcmp(WrdV[1], "friends")==0) { Sign = 1; }
    else if (strcmp(WrdV[1], "fans")==0) { continue; } // skip (fans are in-friends)
    else if (strcmp(WrdV[1], "foes")==0) { Sign = -1; } else { Fail; }
    const int SrcNId = NIdSet.AddKey(WrdV[0]);
    if (! Net->IsNode(SrcNId)) {
      Net->AddNode(SrcNId); }   
    for (int e = 2; e < WrdV.Len(); e++) {
      const int DstNId = NIdSet.AddKey(WrdV[e]);
      i ++ ;
      if ((SrcNId != DstNId) && ! Net->IsEdge(SrcNId, DstNId)) {
        if (! Net->IsNode(DstNId))
          Net->AddNode(DstNId);
        Net->AddEdge(SrcNId, DstNId, Sign);
      }
    }  
  }  
  TSnap::PrintInfo(Net, "Slashdot (" + TInt::GetStr(i) + ")");  

  // copied from gio.h - line 111
  FILE *F = fopen(OutFNm.CStr(), "wt");
  fprintf(F, "# Directed graph: %s\n", OutFNm.CStr());
  if (! Desc.Empty()) 
    fprintf(F, "# %s\n", (Desc).CStr());
    fprintf(F, "# Nodes: %d Edges: %d\n", Net->GetNodes(), Net->GetEdges());
    fprintf(F, "# UserId\tGroupId\tSign\n"); 
  for (TNodeEDatNet<TInt,TInt>::TEdgeI ei = Net->BegEI(); ei < Net->EndEI(); ei++) {
      fprintf(F, "%d\t%d\t%d\n", ei.GetSrcNId(), ei.GetDstNId(), ei()());
  }
  fclose(F);
  
  PrintGraphStatTable(Net, OutFNm, Desc);
}

// Gets the next line to LnChA.
// Returns true, if LnChA contains a valid line.
// Returns false, if LnChA is empty, such as end of file was encountered.
bool TZipIn::GetNextLnBf(TChA& LnChA) {
  int Status;
  int BfN;        // new pointer to the end of line
  int BfP;        // previous pointer to the line start
  LnChA.Clr();
  do {
    if (BfC >= BfL) { BfP = 0; } // reset the current pointer, FindEol() will read a new buffer
    else { BfP = BfC; }
    Status = FindEol(BfN);
    if (Status >= 0) {
      LnChA.AddBf(&Bf[BfP],BfN-BfP);
      if (Status == 1) { return true; } // got a complete line
    }
    // get more data, if the line is incomplete
  } while (Status == 0);
  // eof or the last line has no newline
  return !LnChA.Empty();
}

TStr TCordisEuPart::GetNrOrgNm(const TStr& Str){
  TStrV NrStrV; TChA ChA;
  PSwSet SwSet=TSwSet::GetSwSet(swstEnglish523);
  PSIn HtmlSIn=TStrIn::New(Str);
  THtmlLx Lx(HtmlSIn);
  while (Lx.Sym!=hsyEof){
    if ((Lx.Sym==hsyStr)||(Lx.Sym==hsyNum)){
      Lx.UcChA.ChangeCh('Â', 'A');
      Lx.UcChA.ChangeCh('Á', 'A');
      Lx.UcChA.ChangeCh('Ä', 'A');
      Lx.UcChA.ChangeCh('Ã', 'A');
      Lx.UcChA.ChangeCh('À', 'A');
      Lx.UcChA.ChangeCh('Ç', 'C');
      Lx.UcChA.ChangeCh('É', 'E');
      Lx.UcChA.ChangeCh('È', 'E');
      Lx.UcChA.ChangeCh('Ê', 'E');
      Lx.UcChA.ChangeCh('Í', 'I');
      Lx.UcChA.ChangeCh('Ó', 'O');
      Lx.UcChA.ChangeCh('Ö', 'O');
      Lx.UcChA.ChangeCh('Ñ', 'N');
      Lx.UcChA.ChangeCh('Ü', 'U');
      Lx.UcChA.ChangeCh('Ú', 'U');
      TStr NrStr=THtmlLx::GetAsciiStr(Lx.UcChA);
      if ((NrStr.Len()==1)||(SwSet->IsIn(NrStr))||
       (NrStr=="AG")||(NrStr=="GMBH")||(NrStr=="LTD")||(NrStr=="SRL")||
       (NrStr=="SA")||(NrStr=="EV")||(NrStr=="AS")||(NrStr=="AB")||
       (NrStr=="NV")||(NrStr=="BV")||
       (NrStr=="AKTIENGESELLSCHAFT")||(NrStr=="LIMITED")||
       (NrStr=="FUER")||(NrStr=="FUR")){
        NrStr="";}
      if (!NrStr.Empty()){NrStrV.Add(NrStr);}
    }
    Lx.GetSym();
  }
  TChA NrChA;
  for (int NrStrN=0; NrStrN<NrStrV.Len(); NrStrN++){
    if (NrStrN>0){NrChA+=' ';}
    NrChA+=NrStrV[NrStrN];
  }
  if (NrChA.Empty()){
    NrChA=Str.GetUc();
  }
  return NrChA;
}

/////////////////////////////////////////////////
// Roget-Base
void TRBase::LoadArtfl(const TStr& WebBaseFPath){
  PWebBase WebBase=PWebBase(new TWebMemBase(WebBaseFPath));
  int WebPgP=WebBase->FFirstWebPg(); int WebPgId;
  while (WebBase->FNextWebPg(WebPgP, WebPgId)){
    TStr UrlStr=WebBase->GetUrlStr(WebPgId);
    static TStr RgShStr="RG.sh"; if (!UrlStr.IsStrIn(RgShStr)){continue;}
//    if (!UrlStr.IsStrIn("RG.sh?^544\\")){continue;}

    PWebPg WebPg=WebBase->GetWebPg(WebPgId);
    PSIn SIn=TStrIn::New(WebPg->GetBodyStr());
    PHtmlDoc HtmlDoc=THtmlDoc::New(SIn, hdtAll);
    int TokN=0; PHtmlTok Tok; THtmlLxSym Sym; TStr Str;

    // move to <h2>
    do {HtmlDoc->GetTok(TokN++, Sym, Str);
    } while (!((Sym==hlsyBTag)&&(Str==THtmlTok::H2TagNm)));

    // parse "ddd[A|B]."
    TChA CtgNm; TChA CtgIdNm;
    HtmlDoc->GetTok(TokN++, Sym, Str);
    IAssert(Sym==hlsyNum); CtgNm+=Str; CtgIdNm+=Str;
    HtmlDoc->GetTok(TokN++, Sym, Str);
    if (Sym==hlsyStr){
      IAssert((Str=='A')||(Str=='B')); CtgNm+=Str; CtgIdNm+=Str;
      HtmlDoc->GetTok(TokN++, Sym, Str);
    }
    IAssert((Sym==hlsySSym)&&(Str=='.')); CtgNm+=Str;

    // parse to </h2>"
    TChA BracketStr;
    HtmlDoc->GetTok(TokN++, Sym, Str);
    while (!((Sym==hlsyETag)&&(Str==THtmlTok::H2TagNm))){
      if ((Sym==hlsySSym)&&(Str=='[')){
        HtmlDoc->GetTok(TokN++, Sym, Str);
        while (!((Sym==hlsySSym)&&(Str==']'))){
          if ((!BracketStr.Empty())&&(Sym==hlsyStr)){BracketStr+=' ';}
          BracketStr+=Str; HtmlDoc->GetTok(TokN++, Sym, Str);
        }
        BracketStr.Ins(0, " ["); BracketStr+=']';
      } else {
        if (Sym==hlsyStr){CtgNm+=' ';}
        CtgNm+=Str;
      }
      HtmlDoc->GetTok(TokN++, Sym, Str);
    }
    CtgNm+=BracketStr;
    TNotify::OnNotify(Notify, ntInfo, CtgNm);

    // parse words
    static TStr AdjStr="ADJ"; static TStr AdvStr="ADV";
    static TStr IntStr="INT"; static TStr PgStr="PAGE";
    static TStr PhrStr="PHR"; static TStr PrefStr="PREF";
    static TStr PronStr="PRON";
    HtmlDoc->GetTok(TokN++, Sym, Str);
    IAssert((Sym==hlsyStr)&&((Str=='N')||(Str==AdvStr)));
    while (TokN<HtmlDoc->GetToks()){
      if (Sym==hlsyStr){
        if (Str==PhrStr){break;}
        if ((Str!='N')&&(Str!='V')&&(Str!=AdjStr)&&(Str!=AdvStr)&&
         (Str!=IntStr)&&(Str!=PrefStr)&&(Str!=PronStr)){
          TChA WordStr;
          do {
            if (!WordStr.Empty()){WordStr+=' ';} WordStr+=Str;
            HtmlDoc->GetTok(TokN++, Sym, Str);
          } while (Sym==hlsyStr);
//          TNotify::OnNotify(Notify, ntInfo, WordStr);
        } else {
          HtmlDoc->GetTok(TokN++, Sym, Str);
        }
      } else
      if (Sym==hlsySSym){
        TStr ExpectStr;
        if (Str=='('){ExpectStr=')';}
        else if (Str=='['){ExpectStr=']';}
        else if (Str=='{'){ExpectStr='}';}
        else if (Str=='"'){ExpectStr='"';}
        if (!ExpectStr.Empty()){
          do {HtmlDoc->GetTok(TokN++, Sym, Str);
          } while (!((Sym==hlsySSym)&&(Str==ExpectStr)));
        }
        HtmlDoc->GetTok(TokN++, Sym, Str);
      } else {
        HtmlDoc->GetTok(TokN++, Sym, Str);
      }
    }
  }
}