C++ PhraseTable::clear方法代码示例

void rule_refinement(JKArgs& args)
{
	if(!args.is_set("i"))usage();
	ifstream is(args.value("i").c_str());

	map<string,EntryInfo> srcPhrases,tarPhrases;
	while(is.good())
	{
		string curline="";
		getline(is,curline);
		if(curline=="")break;
		PhraseRuleEntry pre;
		pre.read(curline);
		if(srcPhrases.find(pre.srcPhrase)==srcPhrases.end())
		{
			EntryInfo ei;
			ei.ind=(int)srcPhrases.size();
			ei.score=pre.s2tScore;
			srcPhrases[pre.srcPhrase]=ei;
		}
		else
			srcPhrases[pre.srcPhrase].score+=pre.s2tScore;

		if(tarPhrases.find(pre.tarPhrase)==tarPhrases.end())
		{
			EntryInfo ei;
			ei.ind=(int)tarPhrases.size();
			ei.score=pre.s2tScore;
			tarPhrases[pre.tarPhrase]=ei;
		}
		else
			tarPhrases[pre.tarPhrase].score+=pre.s2tScore;
	}
	is.clear();
	is.seekg(0,ios::beg);
	PhraseTable pt;
	while(is.good())
	{
		string curline="";
		getline(is,curline);
		if(curline==""){pt.print(cout);break;}
		PhraseRuleEntry pre;
		pre.read(curline);
		pre.s2tScore/=srcPhrases[pre.srcPhrase].score;
		pre.t2sScore/=tarPhrases[pre.tarPhrase].score;
		
		if(pt.size()>0&&pt.data().find(pre.srcPhrase)==pt.data().end())
		{
			pt.print(cout);
			pt.clear();
		}
		pt.add(pre.srcPhrase,pre.tarPhrase,pre);
	}
}

int main(int argc, char* argv[])
{
  cerr << "PhraseStatistics v1.1 written by Nicola Bertoldi\n"
       << "modifying PhraseScore v1.4 written by Philipp Koehn\n"
       << "It computes statistics for extracted phrase pairs\n"
       << "if (direct):\n"
       << "src_phrase ||| trg_phrase || freq(src_phrase, trg_phrase) freq(src_phrase) length(src_phrase) length(trg_phrase)\n"
       << "if (inverse)\n"
       << "src_phrase ||| trg_phrase || freq(src_phrase, trg_phrase) freq(trg_phrase) length(src_phrase) length(trg_phrase)\n";

  if (argc != 4 && argc != 5) {
    cerr << "syntax: statistics extract lex phrase-table [inverse]\n";
    exit(1);
  }
  char* &fileNameExtract = argv[1];
  char* &fileNameLex = argv[2];
  char* &fileNamePhraseTable = argv[3];
  inverseFlag = false;
  if (argc > 4) {
    inverseFlag = true;
    cerr << "using inverse mode\n";
  }

  // lexical translation table
  lexTable.load( fileNameLex );

  // sorted phrase extraction file
  Moses::InputFileStream extractFile(fileNameExtract);

  if (extractFile.fail()) {
    cerr << "ERROR: could not open extract file " << fileNameExtract << endl;
    exit(1);
  }
  istream &extractFileP = extractFile;

  // output file: phrase translation table
  phraseTableFile.open(fileNamePhraseTable);
  if (phraseTableFile.fail()) {
    cerr << "ERROR: could not open file phrase table file "
         << fileNamePhraseTable << endl;
    exit(1);
  }

  // loop through all extracted phrase translations
  int lastForeign = -1;
  vector< PhraseAlignment > phrasePairsWithSameF;
  int i=0;

  string line;
  while(getline(extractFileP, line)) {
    if (extractFileP.eof()) break;
    if (++i % 100000 == 0) cerr << "." << flush;

    PhraseAlignment phrasePair;
    bool isPhrasePair = phrasePair.create( line.c_str(), i );
    if (lastForeign >= 0 && lastForeign != phrasePair.foreign) {
      processPhrasePairs( phrasePairsWithSameF );
      for(size_t j=0; j<phrasePairsWithSameF.size(); j++)
        phrasePairsWithSameF[j].clear();
      phrasePairsWithSameF.clear();
      phraseTableE.clear();
      phraseTableF.clear();
      phrasePair.clear(); // process line again, since phrase tables flushed
      phrasePair.create( line.c_str(), i );
      phrasePairBase = 0;
    }
    lastForeign = phrasePair.foreign;
    if (isPhrasePair)
      phrasePairsWithSameF.push_back( phrasePair );
    else
      phrasePairBase++;
  }
  processPhrasePairs( phrasePairsWithSameF );
  phraseTableFile.close();
}

//.........这里部分代码省略.........
		}
		else if (strcmp(argv[i],"--GoodTuring") == 0) {
			goodTuringFlag = true;
			cerr << "using Good Turing discounting\n";
		}
		else if (strcmp(argv[i],"--LogProb") == 0) {
			logProbFlag = true;
			cerr << "using log-probabilities\n";
		}
		else if (strcmp(argv[i],"--NegLogProb") == 0) {
			logProbFlag = true;
			negLogProb = -1;
			cerr << "using negative log-probabilities\n";
		}
		else {
			cerr << "ERROR: unknown option " << argv[i] << endl;
			exit(1);
		}
	}

	// lexical translation table
	if (lexFlag)
		lexTable.load( fileNameLex );
  
	// compute count of counts for Good Turing discounting
	if (goodTuringFlag)
		computeCountOfCounts( fileNameExtract );

	// sorted phrase extraction file
	ifstream extractFile;
	extractFile.open(fileNameExtract);
	if (extractFile.fail()) {
		cerr << "ERROR: could not open extract file " << fileNameExtract << endl;
		exit(1);
	}
	istream &extractFileP = extractFile;

	// output file: phrase translation table
	phraseTableFile.open(fileNamePhraseTable);
	if (phraseTableFile.fail()) 
	{
		cerr << "ERROR: could not open file phrase table file " 
		     << fileNamePhraseTable << endl;
		exit(1);
	}
  
  // loop through all extracted phrase translations
  int lastSource = -1;
  vector< PhraseAlignment > phrasePairsWithSameF;
  int i=0;
	char line[LINE_MAX_LENGTH],lastLine[LINE_MAX_LENGTH];
	lastLine[0] = '\0';
	PhraseAlignment *lastPhrasePair = NULL;
  while(true) {
    if (extractFileP.eof()) break;
    if (++i % 100000 == 0) cerr << "." << flush;
    SAFE_GETLINE((extractFileP), line, LINE_MAX_LENGTH, '\n', __FILE__);
    if (extractFileP.eof())	break;
				
		// identical to last line? just add count
		if (lastSource > 0 && strcmp(line,lastLine) == 0)
		{
			lastPhrasePair->addToCount( line );
			continue;			
		}
		strcpy( lastLine, line );

		// create new phrase pair
		PhraseAlignment phrasePair;
		phrasePair.create( line, i );
		
		// only differs in count? just add count
		if (lastPhrasePair != NULL && lastPhrasePair->equals( phrasePair ))
		{
			lastPhrasePair->count += phrasePair.count;
			phrasePair.clear();
			continue;
		}
		
		// if new source phrase, process last batch
		if (lastSource >= 0 && lastSource != phrasePair.GetSource()) {
			processPhrasePairs( phrasePairsWithSameF );
			for(int j=0;j<phrasePairsWithSameF.size();j++)
				phrasePairsWithSameF[j].clear();
			phrasePairsWithSameF.clear();
			phraseTableT.clear();
			phraseTableS.clear();
			// process line again, since phrase tables flushed
			phrasePair.clear();
			phrasePair.create( line, i ); 
		}
		
		// add phrase pairs to list, it's now the last one
		lastSource = phrasePair.GetSource();
		phrasePairsWithSameF.push_back( phrasePair );
		lastPhrasePair = &phrasePairsWithSameF[phrasePairsWithSameF.size()-1];
	}
	processPhrasePairs( phrasePairsWithSameF );
	phraseTableFile.close();
}

void computeCountOfCounts( char* fileNameExtract )
{
	cerr << "computing counts of counts";
	for(int i=1;i<=GT_MAX;i++) countOfCounts[i] = 0;

	ifstream extractFile;
	extractFile.open( fileNameExtract );
	if (extractFile.fail()) {
		cerr << "ERROR: could not open extract file " << fileNameExtract << endl;
		exit(1);
	}
	istream &extractFileP = extractFile;

	// loop through all extracted phrase translations
	int i=0;
	char line[LINE_MAX_LENGTH],lastLine[LINE_MAX_LENGTH];
	lastLine[0] = '\0';
	PhraseAlignment *lastPhrasePair = NULL;
	while(true) {
		if (extractFileP.eof()) break;
		if (++i % 100000 == 0) cerr << "." << flush;
		SAFE_GETLINE((extractFileP), line, LINE_MAX_LENGTH, '\n', __FILE__);
		if (extractFileP.eof())	break;
		
		// identical to last line? just add count
		if (strcmp(line,lastLine) == 0)
		{
			lastPhrasePair->addToCount( line );
			continue;			
		}
		strcpy( lastLine, line );

		// create new phrase pair
		PhraseAlignment *phrasePair = new PhraseAlignment();
		phrasePair->create( line, i );
		
		if (i == 1)
		{
			lastPhrasePair = phrasePair;
			continue;
		}

		// only differs in count? just add count
		if (lastPhrasePair->match( *phrasePair ))
		{
			lastPhrasePair->count += phrasePair->count;
			phrasePair->clear();
			delete(phrasePair);
			continue;
		}

		// periodically house cleaning
		if (phrasePair->GetSource() != lastPhrasePair->GetSource())
		{
			phraseTableT.clear(); // these would get too big
			phraseTableS.clear(); // these would get too big
			// process line again, since phrase tables flushed
			phrasePair->clear();
			phrasePair->create( line, i ); 
		}

		int count = lastPhrasePair->count + 0.99999;
		if(count <= GT_MAX)
			countOfCounts[ count ]++;
		lastPhrasePair->clear();
		delete( lastPhrasePair );
		lastPhrasePair = phrasePair;
	}
	
	delete lastPhrasePair;

	discountFactor[0] = 0.01; // floor
	cerr << "\n";
	for(int i=1;i<GT_MAX;i++)
	{
		discountFactor[i] = ((float)i+1)/(float)i*(((float)countOfCounts[i+1]+0.1) / ((float)countOfCounts[i]+0.1));
		cerr << "count " << i << ": " << countOfCounts[ i ] << ", discount factor: " << discountFactor[i];
		// some smoothing...
		if (discountFactor[i]>1) 
			discountFactor[i] = 1;
		if (discountFactor[i]<discountFactor[i-1])
			discountFactor[i] = discountFactor[i-1];
		cerr << " -> " << discountFactor[i]*i << endl;
	}
}

//.........这里部分代码省略.........
			cerr << "outputing word alignment in file " << fileNameWordAlignment << endl;
		}
		else if (strcmp(argv[i],"--NoLex") == 0) {
			lexFlag = false;
			cerr << "not computing lexical translation score\n";
		}
		else if (strcmp(argv[i],"--GoodTuring") == 0) {
			goodTuringFlag = true;
			cerr << "using Good Turing discounting\n";
		}
		else if (strcmp(argv[i],"--LogProb") == 0) {
			logProbFlag = true;
			cerr << "using log-probabilities\n";
		}
		else if (strcmp(argv[i],"--NegLogProb") == 0) {
			logProbFlag = true;
			negLogProb = -1;
			cerr << "using negative log-probabilities\n";
		}
		else {
			cerr << "ERROR: unknown option " << argv[i] << endl;
			exit(1);
		}
	}

	// lexical translation table
	if (lexFlag)
		lexTable.load(fileNameLex);
  
	// compute count of counts for Good Turing discounting
	if (goodTuringFlag)
		computeCountOfCounts(fileNameExtract);

	// sorted phrase extraction file
	Bz2LineReader extractFile(fileNameExtract);

	// output file: phrase translation table
	Bz2LineWriter phraseTableFile(fileNamePhraseTable);

	// output word alignment file
	if (!inverseFlag && wordAlignmentFlag) {
		wordAlignmentFile.open(fileNameWordAlignment);
		if (wordAlignmentFile.fail()) {
			cerr << "ERROR: could not open word alignment file "
			     << fileNameWordAlignment << endl;
			exit(1);
		}
	}
  
  // loop through all extracted phrase translations
  int lastSource = -1;
  vector< PhraseAlignment > phrasePairsWithSameF;
  int i=0;
	string lastLine = "";
	PhraseAlignment *lastPhrasePair = NULL;
	for (string line = extractFile.readLine(); !line.empty(); line = extractFile.readLine()) {
		if (line.empty()) break;
		if ((++i)%10000000 == 0) cerr << "[p. score:" << i << "]" << flush;
    else if (i % 100000 == 0) cerr << "." << flush;
		
		// identical to last line? just add count
		if (lastSource >= 0 && line == lastLine) {
			lastPhrasePair->addToCount(line);
			continue;
		}
		lastLine = line;

		// create new phrase pair
		PhraseAlignment phrasePair;
		vector<string> lineVector = tokenize(line.c_str());
		phrasePair.create(lineVector, i);
		
		// only differs in count? just add count
		if (lastPhrasePair != NULL && lastPhrasePair->equals(phrasePair)) {
			lastPhrasePair->count += phrasePair.count;
			phrasePair.clear();
			continue;
		}
		
		// if new source phrase, process last batch
		if (lastSource >= 0 && lastSource != phrasePair.source) {
			processPhrasePairs(phrasePairsWithSameF, phraseTableFile);
			for (size_t j=0; j<phrasePairsWithSameF.size(); phrasePairsWithSameF[j++].clear());
			phrasePairsWithSameF.clear();
			phraseTableT.clear();
			phraseTableS.clear();
			// process line again, since phrase tables flushed
			phrasePair.clear();
			phrasePair.create(lineVector, i);
		}
		
		// add phrase pairs to list, it's now the last one
		lastSource = phrasePair.source;
		phrasePairsWithSameF.push_back(phrasePair);
		lastPhrasePair = &phrasePairsWithSameF[phrasePairsWithSameF.size()-1];
	}
	processPhrasePairs(phrasePairsWithSameF, phraseTableFile);
	if (!inverseFlag && wordAlignmentFlag)
		wordAlignmentFile.close();
}

void computeCountOfCounts(const string& fileNameExtract) {
	if (fileNameExtract == "-") {
		cerr << "The ‘GoodTuring Discounting’ option may not be used with piped input!" << endl;
		exit(9);
	}

	cerr << "computing counts of counts";
	for (size_t i=1; i<=GT_MAX; countOfCounts[i++] = 0);

	Bz2LineReader extractFile(fileNameExtract);

	// loop through all extracted phrase translations
	int i=0;
	string lastLine;
	PhraseAlignment *lastPhrasePair = NULL;

	for (string line = extractFile.readLine(); !line.empty(); line = extractFile.readLine()) {
		if (line.empty()) break;
		if ((++i)%10000000 == 0) cerr << "[" << i << "]" << endl;
    else if (i % 100000 == 0) cerr << "," << flush;
		
		// identical to last line? just add count
		if (line == lastLine) {
			lastPhrasePair->addToCount(line);
			continue;			
		}
		lastLine = line;

		// create new phrase pair
		PhraseAlignment *phrasePair = new PhraseAlignment();
		vector<string> lineVector = tokenize(line.c_str());
		phrasePair->create(lineVector, i);
		
		if (i == 1) {
			lastPhrasePair = phrasePair;
			continue;
		}

		// only differs in count? just add count
		if (lastPhrasePair->match( *phrasePair )) {
			lastPhrasePair->count += phrasePair->count;
			phrasePair->clear();
			delete(phrasePair);
			continue;
		}

		// periodically house cleaning
		if (phrasePair->source != lastPhrasePair->source) {
			phraseTableT.clear(); // these would get too big
			phraseTableS.clear(); // these would get too big
			// process line again, since phrase tables flushed
			phrasePair->clear();
			phrasePair->create(lineVector, i); 
		}

		int count = lastPhrasePair->count + 0.99999;
		if(count <= GT_MAX)
			++countOfCounts[ count ];
		lastPhrasePair->clear();
		delete( lastPhrasePair );
		lastPhrasePair = phrasePair;
	}

	discountFactor[0] = 0.01; // floor
	cerr << "\n";
	for(int i=1;i<GT_MAX; ++i) {
		discountFactor[i] = ((float)i+1)/(float)i*(((float)countOfCounts[i+1]+0.1) / ((float)countOfCounts[i]+0.1));
		cerr << "count " << i << ": " << countOfCounts[ i ] << ", discount factor: " << discountFactor[i];
		// some smoothing...
		if (discountFactor[i]>1) 
			discountFactor[i] = 1;
		if (discountFactor[i]<discountFactor[i-1])
			discountFactor[i] = discountFactor[i-1];
		cerr << " -> " << discountFactor[i]*i << endl;
	}
}