C++ Sentence::size方法代码示例

void ParallelCorpus::PrintSentence(
    const Sentence& sentence, const Vocab& vocab, std::ostream& out) const {
  if (sentence.size() > 0) {
    out << vocab.GetWord(sentence.at(0));
  }
  for (int i = 1; i < sentence.size(); ++i) {
    out << " " << vocab.GetWord(sentence.at(i));
  }
}

void
crf_decode_forward_backward(Sentence & s, CRF_Model & m, vector< map<string, double> > & tagp)
{
  CRF_Sequence cs;
  for (size_t j = 0; j < s.size(); j++) cs.add_state(crfstate(s, j));

  m.decode_forward_backward(cs, tagp);
  //  m.decode_viterbi(cs);

  for (size_t k = 0; k < s.size(); k++) s[k].prd = cs.vs[k].label;
}

    void Run::parse(const std::string &sInputFile, const std::string &sOutputFile,
                    const std::string &sFeatureFile) const {

        Sentence sentence;
        DependencyTree tree;

        std::cout << "Parsing started" << std::endl;

        auto time_begin = time(NULL);

        std::unique_ptr<DepParser> parser(new DepParser(sFeatureFile, sFeatureFile, ParserState::PARSE));
        std::ifstream input(sInputFile);
        std::ofstream output(sOutputFile);
        if (input) {
            while (input >> sentence) {
                if (sentence.size() < MAX_SENTENCE_SIZE) {
                    parser->parse(sentence, &tree);
                    output << tree;
                    tree.clear();
                }
            }
        }
        input.close();
        output.close();

        auto time_end = time(NULL);

        auto seconds = difftime(time_end, time_begin);

        std::cout << "Parsing has finished successfully. Total time taken is: " << difftime(time_end, time_begin) <<
        "s" << std::endl;
    }

bool Predictor::_buildGraph(const Sentence & sen,
		vector<vector<double> > & graph)
{
        vFeaID.clear();
	graph.clear();
	int n = sen.size();
	graph.resize(n, vector<double>(n, 0));
	for(int i = 0; i < n; i++){
		for(int j = 0; j < n; j++){
			if(j == i) continue;
			graph[i][j] = pModel->wordPairWeight(sen, i, j);
		}
	}

	/*vector<int> sid = pModel->getSentenceFeature();
	sort(sid.begin(),sid.end());
	for(size_t i = 0; i < sid.size(); i++)
	{
	        cout<<sid[i]<<" ";
        }
        cout<<endl;
        pModel->resetSentenceFeature();
	int a;
	*/
	//cin>>a;

	return true;
}

realnumber rmse(const Sentence& observation, const vector<realnumber>& prediction) {
    vector<double> squares;
    const size_t dim = prediction.size();
    const size_t nobs = observation.size();
    squares.reserve(2*nobs + dim);

    // we assume that observation is a sparse vector and first
    // assume that all entries are zero and later fix that.
    for (size_t i = 0; i < dim; ++i) {
        squares.push_back(prediction[i] * prediction[i]);
    }

    for (size_t k = 0; k < nobs; ++k) {
        const index_type i = observation[k];
        squares.push_back(-(prediction[i] * prediction[i])); // fix error
        squares.push_back((1-prediction[i]) * (1-prediction[i]));
    }

    std::sort(squares.begin(), squares.end());

    realnumber squared_error = 0;
    for (size_t i = 0; i < squares.size(); ++i) {
        squared_error += squares[i];
    }

    return sqrt(squared_error/prediction.size());
}

	double operator()(const Sentence& ref, const Sentence& hyp)
	{
		const unsigned int MAX_N = 4;
		int numer[MAX_N] = {0, 1, 1, 1};
		int denom[MAX_N] = {0, 1, 1, 1};
		size_t len_ref = ref.size();
		size_t len_hyp = hyp.size();
		map<Sentence, int> possible;
	
		// gather statistics
		for (unsigned int n = 0; n < MAX_N && len_hyp > n; ++n) {
			denom[n] += len_hyp - n;
	
			for (Position k = 0; k + n < len_ref; ++k) {
				++possible[makeNGram(ref, k, n+1)];
			}
	
			for (Position k = 0; k + n < len_hyp; ++k) {
				auto it = possible.find(makeNGram(hyp, k, n+1));
				if (it != possible.end() && it->second > 0) {
					--it->second;
					++numer[n];
				}
			}
		}
	
		if (numer[0] == 0) return 0.0;
	
		// calculate averatge precision
		double np = 0.0;
		for (unsigned int n = 0; n < MAX_N; ++n) {
			np += log((double)numer[n]) - log((double)denom[n]);
		}
	
		// calculate brevity penalty
		double bp = 1.0 - len_ref / (double)len_hyp;
		if (bp > 0.0) bp = 0.0;
	
		return exp(np/MAX_N + bp);
	}

bool Trainer::addBCells(const Sentence & sen, const vector<int> & fa)
{
	vector<int> features;
	for(size_t i = 1; i < sen.size(); i++){
		int j = fa[i];
		int bi = _buildBCell(sen[i].first);
		int bj = _buildBCell(sen[j].first);
		pModel->getFeatureIDVec(sen, j, i, features);
		BCells[bi].addRecFeature(features);
		BCells[bj].addDomFeature(features);
	}
	return true;
}

bool Trainer::rfTrain(const Sentence & sen, const vector<int> & fa)
{
	/*update weights of receptor(features) by learning from a sample*/
	/*construct antigens*/
	for(size_t i = 1; i < sen.size();)
	{
	        _addAntigen(sen, fa,i);
		if(simu->run(sen,fa))
		{
			i++;
		}
	}
	return true;
}

bool Predictor::_buildGraph(const Sentence & sen, 
		vector<vector<double> > & graph)
{	
	graph.clear();
	int n = sen.size();
	graph.resize(n, vector<double>(n, 0));
	for(int i = 0; i < n; i++){
		for(int j = 0; j < n; j++){
			if(j == i) continue;
			graph[i][j] = pModel->wordPairWeight(sen, i, j);
		}
	}
	return true;
}

bool Trainer::_injectAntigen(const Sentence & sen, const std::vector<int> & fa)
{
	for(size_t i = 1; i < sen.size(); i++)
	{
		//int i = 3;
		int j = fa[i];
		_buildAntigen(sen,i,sen[i].first,j,sen[j].first);
		/*clone antigens*/
		cloneAntigens();
	}

	_addAntigen();

	return true;
}

    void Run::parse(const std::string &sInputFile, const std::string &sOutputFile,
                    const std::string &sFeatureFile) const {

        Sentence sentence;
        DependencyTree tree;

        std::cout << "Parsing started" << std::endl;

        std::unique_ptr<DepParser> parser(new DepParser(sFeatureFile, sFeatureFile, ParserState::PARSE));
        std::ifstream input(sInputFile);
        std::ofstream output(sOutputFile);
        if (input) {
            while (input >> sentence) {
                if (sentence.size() < MAX_SENTENCE_SIZE) {
                    parser->parse(sentence, &tree);
                    output << tree;
                    tree.clear();
                }
            }
        }
        input.close();
        output.close();
    }

void readCorpus(const string &fn, SentenceCollection &sc) {
  ifstream f(fn.c_str());
  if (!f.is_open()) {
    cerr << "can't open \"" << fn << endl;
    throw;
  }

  string s;
  Sentence sent;
  while (getline(f, s)) {
    //cerr << "reading line \"" << s << "\"" << endl;
    if ("sent" == s) {
      Token t("SentBegin", makeVariants("SBEG"));
      sent.push_back(t);
      continue;
    } else if ("/sent" == s) {
      Token t("SentEnd", makeVariants("SEND"));
      sent.push_back(t);
      sc.push_back(sent);
      sent.clear();
      continue;
    } else {
      vector<string> fields;
      split(s, '\t', fields);
      if (fields.size() < 2) 
        continue;      

      int id;
      string word;

      stringstream ss(s);
      ss >> id >> word;

      set<MorphInterp> variants;
      for (size_t i = 2; i < fields.size(); i++) {
        if (0 == fields[i].size())
          continue;

        stringstream ss(fields[i]);
        unsigned int lemmaId;
        string lemma;
        ss >> lemmaId >> lemma;
        
        string sgrm;
        string t;
        while (ss >> t) { 
          if (sgrm.size() > 0) sgrm += " ";
          sgrm += t; 
        }

        MorphInterp ts(lemmaId, sgrm); 
        if (0 == ts.size()) {
          cerr << "\"" << s << "\" - \"" << sgrm << "\"" << sgrm.size() << endl;
          throw;
        }
        variants.insert(ts);
      }

      Token t(word, variants);
      sent.push_back(t, id);
    }
  }

  if (sent.size() > 0) 
    sc.push_back(sent);
}