C++ ME_Model类代码示例

void train(ME_Model & model, const string & filename)
{
  ifstream ifile(filename.c_str());
  
  if (!ifile) {
    cerr << "error: cannot open " << filename << endl; 
    exit(1); 
  }

  string line;
  int n = 0;
  while (getline(ifile, line)) {
    vector<Token> vs = read_line(line);
    for (int j = 0; j < (int)vs.size(); j++) {
      ME_Sample mes = sample(vs, j);
      model.add_training_sample(mes);
    }
    if (n++ > 10000) break;
  }    

  model.use_l1_regularizer(1.0);
  //  model.use_l2_regularizer(1.0);
  //  model.use_SGD();
  model.set_heldout(100);
  model.train();
  model.save_to_file("model");
}

int
bidir_train(const vector<Sentence> & vs, int para)
{
  //  vme.clear();
  //  vme.resize(16);

  for (int t = 0; t < 16; t++) {
    if (t != 15 && t != 0) continue;
  //  for (int t = 15; t >= 0; t--) {
    vector<ME_Sample> train;

    if (para != -1 && t % 4 != para) continue;
    //    if (t % 2 == 1) continue;
    cerr << "type = " << t << endl;
    cerr << "extracting features...";
    int n = 0;
    for (vector<Sentence>::const_iterator i = vs.begin(); i != vs.end(); i++) {
      const Sentence & s = *i;
      for (int j = 0; j < s.size(); j++) {

        string pos_left1 = "BOS", pos_left2 = "BOS2";
        if (j >= 1) pos_left1 = s[j-1].pos;
        if (j >= 2) pos_left2 = s[j-2].pos;
        string pos_right1 = "EOS", pos_right2 = "EOS2";
        if (j <= int(s.size()) - 2) pos_right1 = s[j+1].pos;
        if (j <= int(s.size()) - 3) pos_right2 = s[j+2].pos;
        if ( (t & 0x8) == 0 ) pos_left2 = "";
        if ( (t & 0x4) == 0 ) pos_left1 = "";
        if ( (t & 0x2) == 0 ) pos_right1 = "";
        if ( (t & 0x1) == 0 ) pos_right2 = "";

        train.push_back(mesample(s, j, pos_left2, pos_left1, pos_right1, pos_right2));
      }
      //      if (n++ > 1000) break;
    }
    cerr << "done" << endl;

    ME_Model m;
    //    m.set_heldout(1000,0);
    //    m.train(train, 2, 1000, 0);
    m.train(train, 2, 0, 1);
    char buf[1000];
    sprintf(buf, "model.bidir.%d", t);
    m.save_to_file(buf);
    
  }

}

void train(ME_Model & model, const string & input, const string & model_path)
{
    ifstream ifile(input.c_str());

    if (!ifile)
    {
        cerr << "error: cannot open " << input << endl; 
        exit(1);
    }

    string line;
    while (getline(ifile, line))
    {
        vector<string> vs = split(line);
        ME_Sample mes(vs, true);
        model.add_training_sample(mes);
    }

    model.train();
    model.save(model_path);
}

int main(int argc, char** argv)
{
    /*
     * Params: model_file_name, input_file_name, output_file_name
     *
     */
    if (argc < 4)
    {
        exit_with_help();
    }

    string model_path  = argv[1];
    string input_path  = argv[2];
    string output_path = argv[3];

    ME_Model m;
    m.load(model_path);

    validate(m, input_path, output_path);

    return 0;
}

static void
decode_no_context(vector<Token> & vt, const ME_Model & me_none)
{
  int n = vt.size();
  if (n == 0) return;

  for (size_t i = 0; i < n; i++) {
    ME_Sample mes = mesample(vt, i, "", "", "", "");
    me_none.classify(mes);
    vt[i].prd = mes.label;
  }
  
  for (size_t k = 0; k < n; k++) {
    cout << vt[k].str << "/" << vt[k].prd << " ";
  }
  cout << endl;

}

void validate(const ME_Model & model, 
              const string & input_file_name,
              const string & output_file_name)
{
    ifstream ifile(input_file_name.c_str());
    ofstream ofile(output_file_name.c_str());

    if (!ifile)
    {
        cerr << "error: cannot open " << input_file_name << endl;
        exit(1);
    }
    if (!ofile)
    {
        cerr << "error: cannot open " << output_file_name << endl;
        exit(1);
    }

    int n_correct = 0;
    int n_total   = 0;

    string line;
    while (getline(ifile, line))
    {
        vector<string> vs = read_line(line);
        ME_Sample mes = sample(vs);
        model.predict(mes);

        ofile << mes.label << endl;

        if (mes.label == vs[0])  n_correct++;
        n_total++;
    }

    double accuracy = (double)n_correct / n_total;
    cout << "accuracy = " << n_correct << " / " << n_total
         << " = " << accuracy << endl;
}

void test(const ME_Model & model, const string & filename) 
{
  ifstream ifile(filename.c_str());
  
  if (!ifile) {
    cerr << "error: cannot open " << filename << endl; 
    exit(1); 
  }

  int num_correct = 0;
  int num_tokens = 0;
  string line;
  while (getline(ifile, line)) {
    vector<Token> vs = read_line(line);
    for (int j = 0; j < (int)vs.size(); j++) {
      ME_Sample mes = sample(vs, j);
      model.classify(mes);
      if (mes.label == vs[j].pos) num_correct++;
      num_tokens++;
    }
  }    
  cout << "accuracy = " << num_correct << " / " << num_tokens << " = " 
       << (double)num_correct / num_tokens << endl;
}

int main(int argc, char* argv[])
{
	if (argc < 3 || argc > 4) {
		cerr << "Usage: " << argv[0] << "input output [path-to-ruby]" << endl;
		exit(1);
	}

    ME_Model model;

    string inFile = argv[1];
    string outFile = argv[2];
    //string modelFile = argv[3];
    string modelFile = "model1-1.0";
	string rubyCommand = (argc == 4) ? argv[3] : "ruby";

	string eventFile = inFile + ".event";
	string resultFile = inFile + ".result";

    cerr << "Extracting events.";

	string extractionCommand = 
    	rubyCommand + " EventExtracter.rb " + inFile + " " + eventFile;
    system(extractionCommand.c_str());

    cerr << "roading model file." << endl;
    model.load_from_file(modelFile.c_str());
    //model.load_from_file("model" + setID + "-" + ineq);
    //ifstream fileIn(string("/home/users/y-matsu/private/workspace/eclipse-workspace/GENIASS/" + setID + "/test.txt").c_str());
    //ofstream fileOut(string("/home/users/y-matsu/private/workspace/eclipse-workspace/GENIASS/" + setID + "/test-" + ineq + ".prob").c_str());

    ifstream fileIn(eventFile.c_str());
    ofstream fileOut(resultFile.c_str());

    string line, markedTxt;

    getline(fileIn, markedTxt);
    cerr << "start classification." << endl;
    while (getline(fileIn, line)){
        vector<string> tokens;
        split(line, tokens);
        ME_Sample s;

        for(vector<string>::const_iterator token = tokens.begin() + 1;
				token != tokens.end(); ++token){
            s.add_feature(*token);
        }

        (void) model.classify(s);
        fileOut << s.label << endl;
    }
    fileOut.close();
    fileIn.close();

    remove(eventFile.c_str());

	string splitCommand =
    	rubyCommand + " Classifying2Splitting.rb "
		+ resultFile + " " + markedTxt + " " + outFile;

    system(splitCommand.c_str());

	return 0;
}

void
viterbi(vector<Token> & vt, const ME_Model & me)
{
  if (vt.size() == 0) return;
  
  vector< vector<double> > mat;
  vector< vector<int> > bpm;
    
  vector<double> vd(me.num_classes());
  for (size_t j = 0; j < vd.size(); j++) vd[j] = 0;

  mat.push_back(vd);

  for (size_t i = 0; i < vt.size(); i++) {

    vector<double> vd(me.num_classes());
    for (size_t j = 0; j < vd.size(); j++) vd[j] = -999999;
    vector<int> bp(me.num_classes());

    double maxl = -999999;
    for (size_t j = 0; j < vd.size(); j++) {
      if (mat[i][j] > maxl) maxl = mat[i][j];
    }
    
    for (size_t j = 0; j < vd.size(); j++) {
      if (mat[i][j] < maxl - BEAM_WIDTH) continue; // beam thresholding
      
      string prepos = me.get_class_label(j);
      if (i == 0) {
        if (j > 0) continue;
        prepos = "BOS";
      }
      //      prepos = me.get_class_name(j);
      //      if (i == 0 && prepos != "BOS") continue;

      ME_Sample mes = mesample(vt, i, prepos);
      vector<double> membp = me.classify(mes);
      for (size_t k = 0; k < vd.size(); k++) {
        double l = mat[i][j] + log(membp[k]);
        if (l > vd[k]) {
          bp[k] = j;
          vd[k] = l;
        }
      }
    }
    mat.push_back(vd);
    //    for (int k = 0; k < vd.size(); k++) cout << bp[k] << " ";
    //    cout << endl;
    bpm.push_back(bp);
  }
  /*
  for (int i = 0; i < vt.size(); i++) {
    int max_prd = 0;
    for (int j = 0; j < vd.size(); j++) {
      double l = mat[i+1][j];
      if (l > mat[i+1][max_prd]) {
        max_prd = j;
      }
    }
    vt[i].prd = me.get_class_name(max_prd);
  }
  */  

  //  cout << "viterbi ";
  int max_prd = 0;
  int n = vt.size();
  for (size_t j = 0; j < vd.size(); j++) {
    double l = mat[n][j];
    if (l > mat[n][max_prd]) {
      max_prd = j;
    }
  }
  vt[n-1].prd = me.get_class_label(max_prd);
  for (int i = vt.size() - 2; i >= 0; i--) {
    //    cout << max_prd << " ";
    //    cerr << max_prd << " ";
    if (max_prd < 0 || max_prd >= me.num_classes()) exit(0);
    max_prd = bpm[i+1][max_prd];
    vt[i].prd = me.get_class_label(max_prd);
  }
  //  cout << endl;

}

// New model
void new_model() {
    model.clear();
	model = *(new ME_Model());
}

// Print weights
void print_weights() {
	list< pair< pair<string, string>, double > > fl;
	model.get_features(fl);
	for (list< pair< pair<string, string>, double> >::const_iterator i = fl.begin(); i != fl.end(); i++) {
		Rprintf("%10.3f  %-10s %s\n", i->second, i->first.first.c_str(), i->first.second.c_str());
	}
}

// Export weights
vector< vector<string> > export_weights() {
	list< pair< pair<string, string>, double > > fl;
	model.get_features(fl);
	
	vector<string> value1;
	vector<string> value2;
	vector<string> value3;
	for (list< pair< pair<string, string>, double> >::const_iterator i = fl.begin(); i != fl.end(); i++) {
		stringstream write_weights1;
		write_weights1 << setprecision(3) << setw(10) << i->second;
		string weights1 = write_weights1.str();
		value1.push_back(weights1);
		
		stringstream write_weights2;
		write_weights2 << left << setw(10) << i->first.first.c_str();
		string weights2 = write_weights2.str();
		value2.push_back(weights2);
		
		stringstream write_weights3;
		write_weights3 << i->first.second.c_str();
		string weights3 = write_weights3.str();
		value3.push_back(weights3);
	}
	
	vector< vector<string> > results;
	results.push_back(value1);
	results.push_back(value2);
	results.push_back(value3);
	
	return results;
}

void add_samples(int nrows, int ncols, vector<string> samples, vector<double> ia, vector<string> ja, vector<double> ra) {
	new_model();
	for (int i=0; i < nrows; i++) { // for each document
		//Rprintf("Document %d\n",i); // debug output
		ME_Sample newSample(samples[i]); // create new sample for code
		for (int j=ia[i]-1; j < ia[i+1]-1; j++) { // for each feature
			//Rprintf("Feature %s\n",features[j].c_str());
			newSample.add_feature(ja[j],ra[j]);
		}
		model.add_training_sample(newSample);
	}
}

RcppExport SEXP classify_samples(int nrows, int ncols, vector<double> ia, vector<string> ja, vector<double> ra, string model_data) {
	new_model();
	model.load_from_string(model_data);
	vector<string> results;
	vector<string> probability_names;
	
	NumericMatrix probability_matrix(nrows,model.num_classes());
	
	for (int i=0; i < nrows; i++) { // for each document
		//Rprintf("Document %d\n",i); // debug output
		ME_Sample newSample; // create new sample for code
		for (int j=ia[i]-1; j < ia[i+1]-1; j++) { // for each feature
			newSample.add_feature(ja[j],ra[j]);
		}
		
		vector<double> prob = model.classify(newSample);
		for (int k=0; k < model.num_classes(); k++) {
			probability_matrix(i,k) = prob[k];
		}
		
		results.push_back(newSample.label);
	}
	
	for (int k=0; k < model.num_classes(); k++) {
		probability_names.push_back(model.get_class_label(k));
	}
	
	List rs = List::create(results,probability_matrix,probability_names);
	
	return rs;
}

// Train model
RcppExport SEXP train_model(double l1=0, double l2=0, bool sgd=FALSE, int sgd_iter=30, double sgd_eta0=1, double sgd_alpha=0.85, int heldout=0) {
	Rprintf("Training the new model...\n");
	if (heldout > 0) model.set_heldout(heldout);
    if (l1 > 0) model.use_l1_regularizer(l1);
    else if (l2 > 0) model.use_l2_regularizer(l2);
    else if (sgd) model.use_SGD();
	
    model.train();
	
	string model_data = model.save_to_string();
	vector< vector<string> > weights = export_weights();
	List rs = List::create(model_data,weights[0],weights[1],weights[2]);
	
	return rs;
}