Java TransducerEvaluator类代码示例

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
/**
 * Create and train a CRF model from the given training data,
 * optionally testing it on the given test data.
 *
 * @param training training data
 * @param testing test data (possibly <code>null</code>)
 * @param constraints constraints
 * @param crf model
 * @param eval accuracy evaluator (possibly <code>null</code>)
 * @param iterations number of training iterations
 * @param var Gaussian prior variance
 * @param resets Number of resets.
 * @return the trained model
 */
public static CRF trainGE(InstanceList training, InstanceList testing,
    ArrayList<GEConstraint> constraints, CRF crf,
    TransducerEvaluator eval, int iterations, double var, int resets) {

  logger.info("Training on " + training.size() + " instances");
  if (testing != null)
    logger.info("Testing on " + testing.size() + " instances");
  
  assert(numThreads.value > 0);
  
  CRFTrainerByGE trainer = new CRFTrainerByGE(crf,constraints,numThreads.value);
  if (eval != null) trainer.addEvaluator(eval);
  trainer.setGaussianPriorVariance(var);
  trainer.setNumResets(resets);
  trainer.train(training,iterations);

  return crf;
}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
/**
 * Create and train a CRF model from the given training data,
 * optionally testing it on the given test data.
 *
 * @param training training data
 * @param testing test data (possibly <code>null</code>)
 * @param constraints constraints
 * @param crf model
 * @param eval accuracy evaluator (possibly <code>null</code>)
 * @param iterations number of training iterations
 * @param var Gaussian prior variance
 * @return the trained model
 */
public static CRF trainPR(InstanceList training, InstanceList testing,
    ArrayList<PRConstraint> constraints, CRF crf,
    TransducerEvaluator eval, int iterations, double var) {

  logger.info("Training on " + training.size() + " instances");
  if (testing != null)
    logger.info("Testing on " + testing.size() + " instances");
  
  assert(numThreads.value > 0);
  
  CRFTrainerByPR trainer = new CRFTrainerByPR(crf,constraints,numThreads.value);
  trainer.addEvaluator(eval);
  trainer.setPGaussianPriorVariance(var);
  trainer.train(training,iterations,iterations);

  return crf;
}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
public static List<String> run(String trainingFilename, String testingFilename)
        throws FileNotFoundException, IOException {
  ArrayList<Pipe> pipes = new ArrayList<Pipe>();

  pipes.add(new SimpleTaggerSentence2TokenSequence());
  pipes.add(new TokenSequence2FeatureSequence());

  Pipe pipe = new SerialPipes(pipes);

  InstanceList trainingInstances = new InstanceList(pipe);
  InstanceList testingInstances = new InstanceList(pipe);

  trainingInstances.addThruPipe(new LineGroupIterator(
          new BufferedReader(new InputStreamReader(new FileInputStream(trainingFilename))),
          Pattern.compile("^\\s*$"), true));
  testingInstances.addThruPipe(new LineGroupIterator(
          new BufferedReader(new InputStreamReader(new FileInputStream(testingFilename))),
          Pattern.compile("^\\s*$"), true));

  HMM hmm = new HMM(pipe, null);
  hmm.addStatesForLabelsConnectedAsIn(trainingInstances);

  HMMTrainerByLikelihood trainer = new HMMTrainerByLikelihood(hmm);
  TransducerEvaluator testingEvaluator = new SegmentationEvaluator(testingInstances, "testing");
  trainer.train(trainingInstances, 100);
  testingEvaluator.evaluate(trainer);

  return testingInstances.stream().map(Instance::getData).map(Sequence.class::cast)
          .map(hmm::transduce)
          .flatMap(output -> IntStream.range(0, output.size()).mapToObj(output::get))
          .map(String.class::cast).collect(toList());

  // hmm.print();
}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
public void evaluate(File file) throws FileNotFoundException {
	// if (normalize) {
	// normalizeSCLBlock(new InstanceList[] { instances }, limits);
	// }
	InstanceList instances = new InstanceList(pipe);
	instances.addThruPipe(iteratorForFile(file));
	TransducerEvaluator evaluator = new PerClassAccuracyEvaluator(
			instances, "evaluation");
	evaluator.evaluate(trainer);
}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
public boolean trainWithFeatureInduction(InstanceList trainingData,
		InstanceList validationData, InstanceList testingData,
		TransducerEvaluator eval, int numIterations,
		int numIterationsBetweenFeatureInductions,
		int numFeatureInductions, int numFeaturesPerFeatureInduction,
		double trueLabelProbThreshold, boolean clusteredFeatureInduction,
		double[] trainingProportions) {
	return trainWithFeatureInduction(trainingData, validationData,
			testingData, eval, numIterations,
			numIterationsBetweenFeatureInductions, numFeatureInductions,
			numFeaturesPerFeatureInduction, trueLabelProbThreshold,
			clusteredFeatureInduction, trainingProportions, "exp");
}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
public void testDualSpaceViewer () throws IOException
{
  Pipe pipe = TestMEMM.makeSpacePredictionPipe ();
  String[] data0 = { TestCRF.data[0] };
  String[] data1 = TestCRF.data;

  InstanceList training = new InstanceList (pipe);
  training.addThruPipe (new ArrayIterator (data0));
  InstanceList testing = new InstanceList (pipe);
  testing.addThruPipe (new ArrayIterator (data1));

  CRF crf = new CRF (pipe, null);
  crf.addFullyConnectedStatesForLabels ();
  CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf);
  TokenAccuracyEvaluator eval = new TokenAccuracyEvaluator (new InstanceList[] {training, testing}, new String[] {"Training", "Testing"});
  for (int i = 0; i < 5; i++) {
  	crft.train (training, 1);
  	eval.evaluate(crft);
  }

  CRFExtractor extor = hackCrfExtor (crf);
  Extraction e1 = extor.extract (new ArrayIterator (data1));

  Pipe pipe2 = TestMEMM.makeSpacePredictionPipe ();
  InstanceList training2 = new InstanceList (pipe2);
  training2.addThruPipe (new ArrayIterator (data0));
  InstanceList testing2 = new InstanceList (pipe2);
  testing2.addThruPipe (new ArrayIterator (data1));

  MEMM memm = new MEMM (pipe2, null);
  memm.addFullyConnectedStatesForLabels ();
  MEMMTrainer memmt = new MEMMTrainer (memm);
  TransducerEvaluator memmeval = new TokenAccuracyEvaluator (new InstanceList[] {training2, testing2}, new String[] {"Training2", "Testing2"});
  memmt.train (training2, 5);
  memmeval.evaluate(memmt);

  CRFExtractor extor2 = hackCrfExtor (memm);
  Extraction e2 = extor2.extract (new ArrayIterator (data1));

  if (!htmlDir.exists ()) htmlDir.mkdir ();
  LatticeViewer.viewDualResults (htmlDir, e1, extor, e2, extor2);

}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
public void ignoretestDualSpaceViewer () throws IOException
{
  Pipe pipe = TestMEMM.makeSpacePredictionPipe ();
  String[] data0 = { TestCRF.data[0] };
  String[] data1 = TestCRF.data;

  InstanceList training = new InstanceList (pipe);
  training.addThruPipe (new ArrayIterator (data0));
  InstanceList testing = new InstanceList (pipe);
  testing.addThruPipe (new ArrayIterator (data1));

  CRF crf = new CRF (pipe, null);
  crf.addFullyConnectedStatesForLabels ();
  CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf);
  TokenAccuracyEvaluator eval = new TokenAccuracyEvaluator (new InstanceList[] {training, testing}, new String[] {"Training", "Testing"});
  for (int i = 0; i < 5; i++) {
  	crft.train (training, 1);
  	eval.evaluate(crft);
  }

  CRFExtractor extor = hackCrfExtor (crf);
  Extraction e1 = extor.extract (new ArrayIterator (data1));

  Pipe pipe2 = TestMEMM.makeSpacePredictionPipe ();
  InstanceList training2 = new InstanceList (pipe2);
  training2.addThruPipe (new ArrayIterator (data0));
  InstanceList testing2 = new InstanceList (pipe2);
  testing2.addThruPipe (new ArrayIterator (data1));

  MEMM memm = new MEMM (pipe2, null);
  memm.addFullyConnectedStatesForLabels ();
  MEMMTrainer memmt = new MEMMTrainer (memm);
  TransducerEvaluator memmeval = new TokenAccuracyEvaluator (new InstanceList[] {training2, testing2}, new String[] {"Training2", "Testing2"});
  memmt.train (training2, 5);
  memmeval.evaluate(memmt);

  CRFExtractor extor2 = hackCrfExtor (memm);
  Extraction e2 = extor2.extract (new ArrayIterator (data1));

  if (!htmlDir.exists ()) htmlDir.mkdir ();
  LatticeViewer.viewDualResults (htmlDir, e1, extor, e2, extor2);

}
 

import cc.mallet.fst.TransducerEvaluator; //导入依赖的package包/类
/**
 * Test a transducer on the given test data, evaluating accuracy
 * with the given evaluator
 *
 * @param model a <code>Transducer</code>
 * @param eval accuracy evaluator
 * @param testing test data
 */
public static void test(TransducerTrainer tt, TransducerEvaluator eval,
    InstanceList testing)
{
  eval.evaluateInstanceList(tt, testing, "Testing");
}