C++ Vocabulary::storeIfNew方法代码示例

void LexicalTable::load( const string &filePath )
{
  cerr << "Loading lexical translation table from " << filePath;
  ifstream inFile;
  inFile.open(filePath.c_str());
  if (inFile.fail()) {
    cerr << " - ERROR: could not open file\n";
    exit(1);
  }
  istream *inFileP = &inFile;

  string line;

  int i=0;
  while(getline(*inFileP, line)) {
    i++;
    if (i%100000 == 0) cerr << "." << flush;

    vector<string> token = tokenize( line.c_str() );
    if (token.size() != 3) {
      cerr << "line " << i << " in " << filePath << " has wrong number of tokens, skipping:\n" <<
           token.size() << " " << token[0] << " " << line << endl;
      continue;
    }

    double prob = atof( token[2].c_str() );
    WORD_ID wordE = vcbE.storeIfNew( token[0] );
    WORD_ID wordF = vcbF.storeIfNew( token[1] );
    ltable[ wordF ][ wordE ] = prob;
  }
  cerr << endl;
}

bool PhraseAlignment::create(const char line[], int lineID )
{
  vector< string > token = tokenize( line );
  int item = 1;
  PHRASE phraseF, phraseE;
  for (size_t j=0; j<token.size(); j++) {
    if (token[j] == "|||") item++;
    else {
      if (item == 1)
        phraseF.push_back( vcbF.storeIfNew( token[j] ) );
      else if (item == 2)
        phraseE.push_back( vcbE.storeIfNew( token[j] ) );
      else if (item == 3) {
        int e,f;
        sscanf(token[j].c_str(), "%d-%d", &f, &e);
        if ((size_t)e >= phraseE.size() || (size_t)f >= phraseF.size()) {
          cerr << "WARNING: sentence " << lineID << " has alignment point (" << f << ", " << e << ") out of bounds (" << phraseF.size() << ", " << phraseE.size() << ")\n";
        } else {
          if (alignedToE.size() == 0) {
            vector< size_t > dummy;
            for(size_t i=0; i<phraseE.size(); i++)
              alignedToE.push_back( dummy );
            for(size_t i=0; i<phraseF.size(); i++)
              alignedToF.push_back( dummy );
            foreign = phraseTableF.storeIfNew( phraseF );
            english = phraseTableE.storeIfNew( phraseE );
          }
          alignedToE[e].push_back( f );
          alignedToF[f].push_back( e );
        }
      }
    }
  }
  return (item>2); // real phrase pair, not just foreign phrase
}

void LexicalTable::load( char *fileName )
{
  cerr << "Loading lexical translation table from " << fileName;
  ifstream inFile;
  inFile.open(fileName);
  if (inFile.fail()) {
    cerr << " - ERROR: could not open file\n";
    exit(1);
  }
  istream *inFileP = &inFile;

  char line[LINE_MAX_LENGTH];

  int i=0;
  while(true) {
    i++;
    if (i%100000 == 0) cerr << "." << flush;
    SAFE_GETLINE((*inFileP), line, LINE_MAX_LENGTH, '\n', __FILE__);
    if (inFileP->eof()) break;

    vector<string> token = tokenize( line );
    if (token.size() != 3) {
      cerr << "line " << i << " in " << fileName
           << " has wrong number of tokens, skipping:\n"
           << token.size() << " " << token[0] << " " << line << endl;
      continue;
    }

    double prob = atof( token[2].c_str() );
    WORD_ID wordT = vcbT.storeIfNew( token[0] );
    WORD_ID wordS = vcbS.storeIfNew( token[1] );
    ltable[ wordS ][ wordT ] = prob;
  }
  cerr << endl;
}

// read in a phrase pair and store it
void PhraseAlignment::create(const vector<string>& token, int lineID) {
	int item = 1;
	PHRASE phraseS, phraseT;
	for (size_t j=0; j<token.size(); ++j) {
		if (token[j] == "|||")
			item++;
		else if (item == 1) // source phrase
			phraseS.push_back( vcbS.storeIfNew( token[j] ) );
		else if (item == 2) // target phrase
			phraseT.push_back( vcbT.storeIfNew( token[j] ) );
		else if (item == 3) { // alignment
			int s = strtol(token[j].substr(0, token[j].find("-")).c_str(), NULL, 10);
			int t = strtol(token[j].substr(token[j].find("-") + 1).c_str(), NULL, 10);
			if (t >= phraseT.size() || s >= phraseS.size()) {
				cerr << "WARNING: phrase pair " << lineID 
						 << " has alignment point (" << s << ", " << t 
						 << ") out of bounds (" << phraseS.size() << ", " << phraseT.size() << ")\n";
			} else {
				// first alignment point? -> initialize
				if (alignedToT.size() == 0) {
          assert(alignedToS.size() == 0);
          size_t numTgtSymbols = (hierarchicalFlag ? phraseT.size()-1 : phraseT.size());
          alignedToT.resize(numTgtSymbols);
          size_t numSrcSymbols = (hierarchicalFlag ? phraseS.size()-1 : phraseS.size());
          alignedToS.resize(numSrcSymbols);
					source = phraseTableS.storeIfNew( phraseS );
					target = phraseTableT.storeIfNew( phraseT );
				}
				// add alignment point
				alignedToT[t].insert( s );
				alignedToS[s].insert( t );
			}
		} else if (item == 4) // count
			count = strtof(token[j].c_str(), NULL);
	}
	if (item == 3)
		count = 1.0;
	if (item < 3 || item > 4) {
		cerr << "ERROR: faulty line " << lineID << ": ";
		for(vector<string>::const_iterator i = token.begin(); i != token.end(); cerr << *(i++) << " ");
		cerr << endl;
	}
}

void LexicalTable::load(char *fileName) {
  cerr << "Loading lexical translation table from " << fileName;
	Bz2LineReader inFile(fileName, Bz2LineReader::UNCOMPRESSED);
	
	int i = 0;
	for (string line = inFile.readLine(); !line.empty(); line = inFile.readLine()) {
		if (line.empty())
			break;
    if (++i%100000 == 0) cerr << "." << flush;
		
    vector<string> token = tokenize(line.c_str());
    if (token.size() != 3) {
      cerr << "line " << i << " “" << line << "” in " << fileName 
			     << " has wrong number of tokens (" << token.size() << "), skipping:\n"
			     << token.size() << " " << token[0] << " " << line << endl;
      continue;
    }
  
    WORD_ID wordT = vcbT.storeIfNew( token[0] );
    WORD_ID wordS = vcbS.storeIfNew( token[1] );
    ltable[ wordS ][ wordT ] = strtod(token[2].c_str(), NULL);
  }
  cerr << endl;
}