C++ ME_Model::train方法代码示例

void train(ME_Model & model, const string & filename)
{
  ifstream ifile(filename.c_str());
  
  if (!ifile) {
    cerr << "error: cannot open " << filename << endl; 
    exit(1); 
  }

  string line;
  int n = 0;
  while (getline(ifile, line)) {
    vector<Token> vs = read_line(line);
    for (int j = 0; j < (int)vs.size(); j++) {
      ME_Sample mes = sample(vs, j);
      model.add_training_sample(mes);
    }
    if (n++ > 10000) break;
  }    

  model.use_l1_regularizer(1.0);
  //  model.use_l2_regularizer(1.0);
  //  model.use_SGD();
  model.set_heldout(100);
  model.train();
  model.save_to_file("model");
}

// Train model
RcppExport SEXP train_model(double l1=0, double l2=0, bool sgd=FALSE, int sgd_iter=30, double sgd_eta0=1, double sgd_alpha=0.85, int heldout=0) {
	Rprintf("Training the new model...\n");
	if (heldout > 0) model.set_heldout(heldout);
    if (l1 > 0) model.use_l1_regularizer(l1);
    else if (l2 > 0) model.use_l2_regularizer(l2);
    else if (sgd) model.use_SGD();
	
    model.train();
	
	string model_data = model.save_to_string();
	vector< vector<string> > weights = export_weights();
	List rs = List::create(model_data,weights[0],weights[1],weights[2]);
	
	return rs;
}

int
bidir_train(const vector<Sentence> & vs, int para)
{
  //  vme.clear();
  //  vme.resize(16);

  for (int t = 0; t < 16; t++) {
    if (t != 15 && t != 0) continue;
  //  for (int t = 15; t >= 0; t--) {
    vector<ME_Sample> train;

    if (para != -1 && t % 4 != para) continue;
    //    if (t % 2 == 1) continue;
    cerr << "type = " << t << endl;
    cerr << "extracting features...";
    int n = 0;
    for (vector<Sentence>::const_iterator i = vs.begin(); i != vs.end(); i++) {
      const Sentence & s = *i;
      for (int j = 0; j < s.size(); j++) {

        string pos_left1 = "BOS", pos_left2 = "BOS2";
        if (j >= 1) pos_left1 = s[j-1].pos;
        if (j >= 2) pos_left2 = s[j-2].pos;
        string pos_right1 = "EOS", pos_right2 = "EOS2";
        if (j <= int(s.size()) - 2) pos_right1 = s[j+1].pos;
        if (j <= int(s.size()) - 3) pos_right2 = s[j+2].pos;
        if ( (t & 0x8) == 0 ) pos_left2 = "";
        if ( (t & 0x4) == 0 ) pos_left1 = "";
        if ( (t & 0x2) == 0 ) pos_right1 = "";
        if ( (t & 0x1) == 0 ) pos_right2 = "";

        train.push_back(mesample(s, j, pos_left2, pos_left1, pos_right1, pos_right2));
      }
      //      if (n++ > 1000) break;
    }
    cerr << "done" << endl;

    ME_Model m;
    //    m.set_heldout(1000,0);
    //    m.train(train, 2, 1000, 0);
    m.train(train, 2, 0, 1);
    char buf[1000];
    sprintf(buf, "model.bidir.%d", t);
    m.save_to_file(buf);
    
  }

}

void train(ME_Model & model, const string & input, const string & model_path)
{
    ifstream ifile(input.c_str());

    if (!ifile)
    {
        cerr << "error: cannot open " << input << endl; 
        exit(1);
    }

    string line;
    while (getline(ifile, line))
    {
        vector<string> vs = split(line);
        ME_Sample mes(vs, true);
        model.add_training_sample(mes);
    }

    model.train();
    model.save(model_path);
}