Java MEMM类代码示例

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void testGetSetParameters()
{
  int inputVocabSize = 100;
  int numStates = 5;
  Alphabet inputAlphabet = new Alphabet();
  for (int i = 0; i < inputVocabSize; i++)
    inputAlphabet.lookupIndex("feature" + i);
  Alphabet outputAlphabet = new Alphabet();
  MEMM memm = new MEMM (inputAlphabet, outputAlphabet);
  String[] stateNames = new String[numStates];
  for (int i = 0; i < numStates; i++)
    stateNames[i] = "state" + i;
  memm.addFullyConnectedStates(stateNames);
  MEMMTrainer memmt = new MEMMTrainer (memm);
  MEMMTrainer.MEMMOptimizableByLabelLikelihood omemm = memmt.getOptimizableMEMM (new InstanceList(null));
  TestOptimizable.testGetSetParameters(omemm);
}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void testSpaceSerializable () throws IOException, ClassNotFoundException
{
  Pipe p = makeSpacePredictionPipe ();
  InstanceList training = new InstanceList (p);
  training.addThruPipe (new ArrayIterator (data));

  MEMM memm = new MEMM (p, null);
  memm.addFullyConnectedStatesForLabels ();
  memm.addStartState();
  memm.setWeightsDimensionAsIn(training);
 MEMMTrainer memmt = new MEMMTrainer (memm);
  memmt.train (training, 10);

  MEMM memm2 = (MEMM) TestSerializable.cloneViaSerialization (memm);

  Optimizable.ByGradientValue mcrf1 = memmt.getOptimizableMEMM(training);
  double val1 = mcrf1.getValue ();
  Optimizable.ByGradientValue mcrf2 = memmt.getOptimizableMEMM(training);
  double val2 = mcrf2.getValue ();

  assertEquals (val1, val2, 1e-5);
}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void disabledtestPrint ()
{
	Pipe p = new SerialPipes (new Pipe[] {
     new CharSequence2TokenSequence("."),
		 new TokenText(),
		 new TestMEMM.TestMEMMTokenSequenceRemoveSpaces(),
		 new TokenSequence2FeatureVectorSequence(),
		 new PrintInputAndTarget(),
  });
	InstanceList one = new InstanceList (p);
	String[] data = new String[] { "ABCDE", };
	one.addThruPipe (new ArrayIterator (data));
	MEMM crf = new MEMM (p, null);
	crf.addFullyConnectedStatesForLabels();
	crf.setWeightsDimensionAsIn (one);
	MEMMTrainer memmt = new MEMMTrainer (crf);
	MEMMTrainer.MEMMOptimizableByLabelLikelihood mcrf = memmt.getOptimizableMEMM(one);
	double[] params = new double[mcrf.getNumParameters()];
	for (int i = 0; i < params.length; i++) {
		params [i] = i;
	}
	mcrf.setParameters (params);
	crf.print ();
}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void disabledtestPrint ()
{
	Pipe p = new SerialPipes (new Pipe[] {
     new CharSequence2TokenSequence("."),
		 new TokenText(),
		 new TestMEMMTokenSequenceRemoveSpaces(),
		 new TokenSequence2FeatureVectorSequence(),
		 new PrintInputAndTarget(),
  });
	InstanceList one = new InstanceList (p);
	String[] data = new String[] { "ABCDE", };
	one.addThruPipe (new ArrayIterator (data));
	MEMM crf = new MEMM (p, null);
	crf.addFullyConnectedStatesForLabels();
	crf.setWeightsDimensionAsIn (one);
	MEMMTrainer memmt = new MEMMTrainer (crf);
	MEMMTrainer.MEMMOptimizableByLabelLikelihood mcrf = memmt.getOptimizableMEMM(one);
	double[] params = new double[mcrf.getNumParameters()];
	for (int i = 0; i < params.length; i++) {
		params [i] = i;
	}
	mcrf.setParameters (params);
	crf.print ();
}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void ignoretestSpaceSerializable () throws IOException, ClassNotFoundException
{
  Pipe p = makeSpacePredictionPipe ();
  InstanceList training = new InstanceList (p);
  training.addThruPipe (new ArrayIterator (data));

  MEMM memm = new MEMM (p, null);
  memm.addFullyConnectedStatesForLabels ();
  memm.addStartState();
  memm.setWeightsDimensionAsIn(training);
 MEMMTrainer memmt = new MEMMTrainer (memm);
  memmt.train (training, 10);

  MEMM memm2 = (MEMM) TestSerializable.cloneViaSerialization (memm);

  Optimizable.ByGradientValue mcrf1 = memmt.getOptimizableMEMM(training);
  double val1 = mcrf1.getValue ();
  Optimizable.ByGradientValue mcrf2 = memmt.getOptimizableMEMM(training);
  double val2 = mcrf2.getValue ();

  assertEquals (val1, val2, 1e-5);
}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void testSpaceMaximizable ()
  {
    Pipe p = makeSpacePredictionPipe ();
    InstanceList training = new InstanceList (p);
//    String[] data = { TestMEMM.data[0], }; // TestMEMM.data[1], TestMEMM.data[2], TestMEMM.data[3], };
//    String[] data = { "ab" };
    training.addThruPipe (new ArrayIterator (data));

//    CRF4 memm = new CRF4 (p, null);
    MEMM memm = new MEMM (p, null);
    memm.addFullyConnectedStatesForLabels ();
    memm.addStartState();
    memm.setWeightsDimensionAsIn(training);
    
	  MEMMTrainer memmt = new MEMMTrainer (memm);
//    memm.gatherTrainingSets (training); // ANNOYING: Need to set up per-instance training sets
    memmt.train (training, 1);  // Set weights dimension, gathers training sets, etc.

//    memm.print();
//    memm.printGradient = true;
//    memm.printInstanceLists();

//    memm.setGaussianPriorVariance (Double.POSITIVE_INFINITY);
    Optimizable.ByGradientValue mcrf = memmt.getOptimizableMEMM(training);
    TestOptimizable.setNumComponents (150);
    TestOptimizable.testValueAndGradient (mcrf);
  }
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void ignoretestSpaceMaximizable ()
  {
    Pipe p = makeSpacePredictionPipe ();
    InstanceList training = new InstanceList (p);
//    String[] data = { TestMEMM.data[0], }; // TestMEMM.data[1], TestMEMM.data[2], TestMEMM.data[3], };
//    String[] data = { "ab" };
    training.addThruPipe (new ArrayIterator (data));

//    CRF4 memm = new CRF4 (p, null);
    MEMM memm = new MEMM (p, null);
    memm.addFullyConnectedStatesForLabels ();
    memm.addStartState();
    memm.setWeightsDimensionAsIn(training);
    
	  MEMMTrainer memmt = new MEMMTrainer (memm);
//    memm.gatherTrainingSets (training); // ANNOYING: Need to set up per-instance training sets
    memmt.train (training, 1);  // Set weights dimension, gathers training sets, etc.

//    memm.print();
//    memm.printGradient = true;
//    memm.printInstanceLists();

//    memm.setGaussianPriorVariance (Double.POSITIVE_INFINITY);
    Optimizable.ByGradientValue mcrf = memmt.getOptimizableMEMM(training);
    TestOptimizable.setNumComponents (150);
    TestOptimizable.testValueAndGradient (mcrf);
  }
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void disabledtestAddOrderNStates ()
{
   Pipe p = makeSpacePredictionPipe ();

   InstanceList instances = new InstanceList (p);
  instances.addThruPipe (new ArrayIterator(data));
  InstanceList[] lists = instances.split (new java.util.Random (678), new double[]{.5, .5});

	// Compare 3 CRFs trained with addOrderNStates, and make sure
	// that having more features leads to a higher likelihood

  MEMM crf1 = new MEMM(p.getDataAlphabet(), p.getTargetAlphabet());
  crf1.addOrderNStates (lists [0],
											 new int[] { 1, },
											 new boolean[] { false, },
											 "START",
											 null,
											 null,
											 false);
  crf1.setWeightsDimensionAsIn(lists[0]);
  MEMMTrainer memmt1 = new MEMMTrainer (crf1);
	memmt1.train(lists [0]);


  MEMM crf2 = new MEMM(p.getDataAlphabet(), p.getTargetAlphabet());
  crf2.addOrderNStates (lists [0],
												 new int[] { 1, 2, },
												 new boolean[] { false, true },
												 "START",
												 null,
												 null,
												 false);
  crf2.setWeightsDimensionAsIn(lists[0]);
  MEMMTrainer memmt2 = new MEMMTrainer (crf2);
	memmt2.train(lists [0]);


  MEMM crf3 = new MEMM(p.getDataAlphabet(), p.getTargetAlphabet());
  crf3.addOrderNStates (lists [0],
											 new int[] { 1, 2, },
											 new boolean[] { false, false },
											 "START",
											 null,
											 null,
											 false);
  crf3.setWeightsDimensionAsIn(lists[0]);
  MEMMTrainer memmt3 = new MEMMTrainer (crf3);
	memmt3.train(lists [0]);

	// Prevent cached values
	double lik1 = getLikelihood (memmt1, lists[0]);
	double lik2 = getLikelihood (memmt2, lists[0]);
	double lik3 = getLikelihood (memmt3, lists[0]);

	System.out.println("CRF1 likelihood "+lik1);

	assertTrue ("Final zero-order likelihood <"+lik1+"> greater than first-order <"+lik2+">",
							lik1 < lik2);
	assertTrue ("Final defaults-only likelihood <"+lik2+"> greater than full first-order <"+lik3+">",
							lik2 < lik3);

	assertEquals (-167.335971702, lik1, 0.0001);
	assertEquals (-166.212235389, lik2, 0.0001);
	assertEquals ( -90.386005741, lik3, 0.0001);
}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void testDualSpaceViewer () throws IOException
{
  Pipe pipe = TestMEMM.makeSpacePredictionPipe ();
  String[] data0 = { TestCRF.data[0] };
  String[] data1 = TestCRF.data;

  InstanceList training = new InstanceList (pipe);
  training.addThruPipe (new ArrayIterator (data0));
  InstanceList testing = new InstanceList (pipe);
  testing.addThruPipe (new ArrayIterator (data1));

  CRF crf = new CRF (pipe, null);
  crf.addFullyConnectedStatesForLabels ();
  CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf);
  TokenAccuracyEvaluator eval = new TokenAccuracyEvaluator (new InstanceList[] {training, testing}, new String[] {"Training", "Testing"});
  for (int i = 0; i < 5; i++) {
  	crft.train (training, 1);
  	eval.evaluate(crft);
  }

  CRFExtractor extor = hackCrfExtor (crf);
  Extraction e1 = extor.extract (new ArrayIterator (data1));

  Pipe pipe2 = TestMEMM.makeSpacePredictionPipe ();
  InstanceList training2 = new InstanceList (pipe2);
  training2.addThruPipe (new ArrayIterator (data0));
  InstanceList testing2 = new InstanceList (pipe2);
  testing2.addThruPipe (new ArrayIterator (data1));

  MEMM memm = new MEMM (pipe2, null);
  memm.addFullyConnectedStatesForLabels ();
  MEMMTrainer memmt = new MEMMTrainer (memm);
  TransducerEvaluator memmeval = new TokenAccuracyEvaluator (new InstanceList[] {training2, testing2}, new String[] {"Training2", "Testing2"});
  memmt.train (training2, 5);
  memmeval.evaluate(memmt);

  CRFExtractor extor2 = hackCrfExtor (memm);
  Extraction e2 = extor2.extract (new ArrayIterator (data1));

  if (!htmlDir.exists ()) htmlDir.mkdir ();
  LatticeViewer.viewDualResults (htmlDir, e1, extor, e2, extor2);

}
 

import cc.mallet.fst.MEMM; //导入依赖的package包/类
public void ignoretestDualSpaceViewer () throws IOException
{
  Pipe pipe = TestMEMM.makeSpacePredictionPipe ();
  String[] data0 = { TestCRF.data[0] };
  String[] data1 = TestCRF.data;

  InstanceList training = new InstanceList (pipe);
  training.addThruPipe (new ArrayIterator (data0));
  InstanceList testing = new InstanceList (pipe);
  testing.addThruPipe (new ArrayIterator (data1));

  CRF crf = new CRF (pipe, null);
  crf.addFullyConnectedStatesForLabels ();
  CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood (crf);
  TokenAccuracyEvaluator eval = new TokenAccuracyEvaluator (new InstanceList[] {training, testing}, new String[] {"Training", "Testing"});
  for (int i = 0; i < 5; i++) {
  	crft.train (training, 1);
  	eval.evaluate(crft);
  }

  CRFExtractor extor = hackCrfExtor (crf);
  Extraction e1 = extor.extract (new ArrayIterator (data1));

  Pipe pipe2 = TestMEMM.makeSpacePredictionPipe ();
  InstanceList training2 = new InstanceList (pipe2);
  training2.addThruPipe (new ArrayIterator (data0));
  InstanceList testing2 = new InstanceList (pipe2);
  testing2.addThruPipe (new ArrayIterator (data1));

  MEMM memm = new MEMM (pipe2, null);
  memm.addFullyConnectedStatesForLabels ();
  MEMMTrainer memmt = new MEMMTrainer (memm);
  TransducerEvaluator memmeval = new TokenAccuracyEvaluator (new InstanceList[] {training2, testing2}, new String[] {"Training2", "Testing2"});
  memmt.train (training2, 5);
  memmeval.evaluate(memmt);

  CRFExtractor extor2 = hackCrfExtor (memm);
  Extraction e2 = extor2.extract (new ArrayIterator (data1));

  if (!htmlDir.exists ()) htmlDir.mkdir ();
  LatticeViewer.viewDualResults (htmlDir, e1, extor, e2, extor2);

}