C++ TargetPhrase::SetScore方法代码示例

//.........这里部分代码省略.........
            
            // set subphrase by default to itself
            TargetPhraseVectorPtr subTpv = tpv;
            
            // if range smaller than source phrase retrieve subphrase
            if(unsigned(srcEnd - srcStart + 1) != srcSize)
            {
              Phrase subPhrase = sourcePhrase.GetSubString(WordsRange(srcStart, srcEnd));
              subTpv = CreateTargetPhraseCollection(subPhrase, false);
            }
            
            // false positive consistency check
            if(subTpv != NULL && rank < subTpv->size())
            {
              // insert the subphrase into the main target phrase
              TargetPhrase& subTp = subTpv->at(rank);
              if(m_phraseDictionary.m_useAlignmentInfo)
              {
                // reconstruct the alignment data based on the alignment of the subphrase
                for(AlignmentInfo::const_iterator it = subTp.GetAlignmentInfo().begin();
                    it != subTp.GetAlignmentInfo().end(); it++)
                {
                  alignment.insert(AlignPointSizeT(srcStart + it->first,
                                                   targetPhrase->GetSize() + it->second));
                }
              }
              targetPhrase->Append(subTp);
            }
            else 
              return TargetPhraseVectorPtr();
          }
        }
        else
        {
            Word word;
            word.CreateFromString(Output, *m_output,
                                  GetTargetSymbol(symbol), false);
            targetPhrase->AddWord(word);
        }
      }
    }
    else if(state == Score)
    {
      size_t idx = m_multipleScoreTrees ? scores.size() : 0;
      float score = m_scoreTrees[idx]->Read(encodedBitStream);
      scores.push_back(score);
      
      if(scores.size() == m_numScoreComponent)
      {
        targetPhrase->SetScore(m_feature, scores, ScoreComponentCollection() /*sparse*/,*m_weight, m_weightWP, *m_languageModels);
        
        if(m_containsAlignmentInfo)
          state = Alignment;
        else
          state = Add;
      }
    }
    else if(state == Alignment)
    {
      AlignPoint alignPoint = m_alignTree->Read(encodedBitStream);
      if(alignPoint == alignStopSymbol)
      {
        state = Add;
      }
      else
      {
        if(m_phraseDictionary.m_useAlignmentInfo)  
          alignment.insert(AlignPointSizeT(alignPoint));
      }
    }
    
    if(state == Add)
    {
      if(m_phraseDictionary.m_useAlignmentInfo)
        targetPhrase->SetAlignmentInfo(alignment);
      
      if(m_coding == PREnc)
      {
        if(!m_maxRank || tpv->size() <= m_maxRank)
          bitsLeft = encodedBitStream.TellFromEnd();
        
        if(!topLevel && m_maxRank && tpv->size() >= m_maxRank)
          break;
      }
      
      if(encodedBitStream.TellFromEnd() <= 8)
        break;
      
      state = New;
    }    
  }
  
  if(m_coding == PREnc && !extending)
  {
    bitsLeft = bitsLeft > 8 ? bitsLeft : 0;
    m_decodingCache.Cache(sourcePhrase, tpv, bitsLeft, m_maxRank);
  }
  
  return tpv;
}

OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
{
  size_t scoreInd = 0;

  // MAIN LOOP
  size_t stage = 0;
  /*	0 = source phrase
   1 = target phrase
   2 = scores
   3 = align
   4 = count
   */
  char *tok = strtok (line," ");
  OnDiskPt::PhrasePtr out(new Phrase());
  while (tok != NULL) {
    if (0 == strcmp(tok, "|||")) {
      ++stage;
    } else {
      switch (stage) {
      case 0: {
        WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper, 1);
        if (w != NULL)
          out->AddWord(w);

        break;
      }
      case 1: {
        Tokenize(targetPhrase, tok, false, true, onDiskWrapper, 0);
        break;
      }
      case 2: {
        float score = Moses::Scan<float>(tok);
        targetPhrase.SetScore(score, scoreInd);
        ++scoreInd;
        break;
      }
      case 3: {
        //targetPhrase.Create1AlignFromString(tok);
        targetPhrase.CreateAlignFromString(tok);
        break;
      }
      case 4:
        ++stage;
        break;
        /*      case 5: {
              // count info. Only store the 2nd one
              float val = Moses::Scan<float>(tok);
              misc[0] = val;
              ++stage;
              break;
        }*/
      case 5: {
        // count info. Only store the 2nd one
        //float val = Moses::Scan<float>(tok);
        //misc[0] = val;
        ++stage;
        break;
      }
      case 6: {
        // store only the 3rd one (rule count)
        float val = Moses::Scan<float>(tok);
        misc[0] = val;
        ++stage;
        break;
      }
      default:
        cerr << "ERROR in line " << line << endl;
        assert(false);
        break;
      }
    }

    tok = strtok (NULL, " ");
  } // while (tok != NULL)

  assert(scoreInd == numScores);
  targetPhrase.SortAlign();
  return out;
} // Tokenize()

void Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
{
	size_t scoreInd = 0;
	
	// MAIN LOOP
	size_t stage = 0;
	/*	0 = source phrase
	 1 = target phrase
	 2 = scores
	 3 = align
	 4 = count
	 */
	char *tok = strtok (line," ");
	while (tok != NULL)
	{
		if (0 == strcmp(tok, "|||"))
		{
			++stage;
		}
		else
		{
			switch (stage)
			{
				case 0:
				{
					Tokenize(sourcePhrase, tok, true, true, onDiskWrapper);
					break;
				}
				case 1:
				{
					Tokenize(targetPhrase, tok, false, true, onDiskWrapper);
					break;
				}
				case 2:
				{
					float score = Moses::Scan<float>(tok);
					targetPhrase.SetScore(score, scoreInd);
					++scoreInd;
					break;
				}
				case 3:
				{
					targetPhrase.Create1AlignFromString(tok);
					break;
				}
				case 4:
					++stage;
					break;
				case 5:					
				{ // count info. Only store the 2nd one
					float val = Moses::Scan<float>(tok);
					misc[0] = val;
					++stage;
					break;
				}
				default:
					assert(false);
					break;
			}
		}
		
		tok = strtok (NULL, " ");
	} // while (tok != NULL)
	
	assert(scoreInd == numScores);
	targetPhrase.SortAlign();
	
} // Tokenize()

OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, const std::string &lineStr, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc)
{
  char line[lineStr.size() + 1];
  strcpy(line, lineStr.c_str());

  stringstream sparseFeatures, property;

  size_t scoreInd = 0;

  // MAIN LOOP
  size_t stage = 0;
  /*	0 = source phrase
   1 = target phrase
   2 = scores
   3 = align
   4 = count
   7 = properties
   */
  char *tok = strtok (line," ");
  OnDiskPt::PhrasePtr out(new Phrase());
  while (tok != NULL) {
    if (0 == strcmp(tok, "|||")) {
      ++stage;
    } else {
      switch (stage) {
      case 0: {
        WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper, 1);
        if (w != NULL)
          out->AddWord(w);

        break;
      }
      case 1: {
        Tokenize(targetPhrase, tok, false, true, onDiskWrapper, 0);
        break;
      }
      case 2: {
        float score = Moses::Scan<float>(tok);
        targetPhrase.SetScore(score, scoreInd);
        ++scoreInd;
        break;
      }
      case 3: {
        //targetPhrase.Create1AlignFromString(tok);
        targetPhrase.CreateAlignFromString(tok);
        break;
      }
      case 4: {
        // store only the 3rd one (rule count)
        float val = Moses::Scan<float>(tok);
        misc[0] = val;
        break;
      }
      case 5: {
        // sparse features
        sparseFeatures << tok << " ";
        break;
      }
      case 6: {
        property << tok << " ";
        break;
      }
      default:
        cerr << "ERROR in line " << line << endl;
        assert(false);
        break;
      }
    }

    tok = strtok (NULL, " ");
  } // while (tok != NULL)

  assert(scoreInd == numScores);
  targetPhrase.SetSparseFeatures(Moses::Trim(sparseFeatures.str()));
  targetPhrase.SetProperty(Moses::Trim(property.str()));
  targetPhrase.SortAlign();
  return out;
} // Tokenize()

  const TargetPhraseCollection*
     PhraseDictionaryInterpolated::GetTargetPhraseCollection(const Phrase& src) const {

    delete m_targetPhrases;
    m_targetPhrases = new TargetPhraseCollection();
    PhraseSet allPhrases;
    vector<PhraseSet> phrasesByTable(m_dictionaries.size());
    for (size_t i = 0; i < m_dictionaries.size(); ++i) {
      const TargetPhraseCollection* phrases = m_dictionaries[i]->GetTargetPhraseCollection(src);
      if (phrases) {
        for (TargetPhraseCollection::const_iterator j = phrases->begin(); 
          j != phrases->end(); ++j) {
          allPhrases.insert(*j);
          phrasesByTable[i].insert(*j);
        }
      }
    }
    ScoreComponentCollection sparseVector;
    for (PhraseSet::const_iterator i = allPhrases.begin(); i != allPhrases.end(); ++i) {
      TargetPhrase* combinedPhrase = new TargetPhrase((Phrase)**i);
      //combinedPhrase->ResetScore();
      //cerr << *combinedPhrase << " " << combinedPhrase->GetScoreBreakdown() << endl;
      combinedPhrase->SetSourcePhrase((*i)->GetSourcePhrase());
      combinedPhrase->SetAlignTerm(&((*i)->GetAlignTerm()));
      combinedPhrase->SetAlignNonTerm(&((*i)->GetAlignTerm()));
      Scores combinedScores(GetFeature()->GetNumScoreComponents());
      for (size_t j = 0; j < phrasesByTable.size(); ++j) {
        PhraseSet::const_iterator tablePhrase = phrasesByTable[j].find(combinedPhrase);
        if (tablePhrase != phrasesByTable[j].end()) {
          Scores tableScores = (*tablePhrase)->GetScoreBreakdown()
            .GetScoresForProducer(GetFeature());
          //cerr << "Scores from " << j << " table: ";
          for (size_t k = 0; k < tableScores.size()-1; ++k) {
            //cerr << tableScores[k] << "(" << exp(tableScores[k]) << ") ";
            combinedScores[k] += m_weights[k][j] * exp(tableScores[k]);
            //cerr << m_weights[k][j] * exp(tableScores[k]) << " ";
          }
          //cerr << endl;
        }
      }
      //map back to log space
      //cerr << "Combined ";
      for (size_t k = 0; k < combinedScores.size()-1; ++k) {
        //cerr << combinedScores[k] << " ";
        combinedScores[k] = log(combinedScores[k]);
        //cerr << combinedScores[k] << " ";
      }
      //cerr << endl;
      combinedScores.back() = 1; //assume last is penalty
      combinedPhrase->SetScore(
        GetFeature(),
        combinedScores,
        sparseVector,
        m_weightT,
        m_weightWP,
        *m_languageModels);
      //cerr << *combinedPhrase << " " << combinedPhrase->GetScoreBreakdown() <<  endl;
      m_targetPhrases->Add(combinedPhrase);
    }

    m_targetPhrases->Prune(true,m_tableLimit);


    return m_targetPhrases;
  }