Java CRFTrainerByThreadedLabelLikelihood类代码示例

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
public CRF train() {
	t = 0;
	do {
		crf = new CRF(crf.getInputPipe(), crf.getOutputPipe());
		crf.addStatesForBiLabelsConnectedAsIn(Ds);
		trainer = new CRFTrainerByThreadedLabelLikelihood(crf, getThreads());
		trainer.train(Ds);
		trainer.shutdown();
		runEvaluators();
		t++;
		System.err.println("Finished iteration " + t);
		if (Rt.isEmpty())
			break;
		label();
		select();
	} while (!stop);
	return crf;
}
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
private TransducerTrainer trainOnce(Pipe pipe, InstanceList trainData) {
  Stopwatch watch = Stopwatch.createStarted();

  CRF crf = new CRF(pipe, null);
  crf.addOrderNStates(trainData, new int[]{1}, null, null, null, null, false);
  crf.addStartState();
  crf.setWeightsDimensionAsIn(trainData, false);
  if (initFrom != null) {
    crf.initializeApplicableParametersFrom(initFrom);
  }

  log.info("Starting alignTag training...");
  CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(crf, 8);
  trainer.setGaussianPriorVariance(2);
  trainer.setAddNoFactors(true);
  trainer.setUseSomeUnsupportedTrick(false);
  trainer.train(trainData);
  trainer.shutdown();
  watch.stop();

  log.info("Syll align Tag CRF Training took " + watch.toString());
  crf.getInputAlphabet().stopGrowth();
  crf.getOutputAlphabet().stopGrowth();
  return trainer;
}
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
private TransducerTrainer trainOnce(Pipe pipe, InstanceList examples) {
    Stopwatch watch = Stopwatch.createStarted();

    CRF crf = new CRF(pipe, null);
    crf.addOrderNStates(examples, new int[]{1}, null, null, null, null, false);
    crf.addStartState();
//    crf.setWeightsDimensionAsIn(examples, false);

    log.info("Starting syllchain training...");
    CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(crf, 8);
    trainer.setGaussianPriorVariance(2);
//    trainer.setUseSomeUnsupportedTrick(false);
//    trainer.setAddNoFactors(true);
    trainer.train(examples);
    trainer.shutdown();
    watch.stop();

    log.info("SyllChain CRF Training took " + watch.toString());
    crf.getInputAlphabet().stopGrowth();
    crf.getOutputAlphabet().stopGrowth();
    return trainer;
  }
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
private TransducerTrainer trainOnce(Pipe pipe, InstanceList trainData) {
    Stopwatch watch = Stopwatch.createStarted();

    CRF crf = new CRF(pipe, null);
    crf.addOrderNStates(trainData, new int[]{1}, null, null, null, null, false);
    crf.addStartState();

    log.info("Starting alignTag training...");
    CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(crf, 8);
    trainer.setGaussianPriorVariance(2);
//    trainer.setUseSomeUnsupportedTrick(false);
    trainer.train(trainData);
    trainer.shutdown();
    watch.stop();

    log.info("Align Tag CRF Training took " + watch.toString());
    crf.getInputAlphabet().stopGrowth();
    crf.getOutputAlphabet().stopGrowth();
    return trainer;
  }
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
private TransducerTrainer trainOnce(Pipe pipe, InstanceList examples) {
    Stopwatch watch = Stopwatch.createStarted();

    CRF crf = new CRF(pipe, null);
    crf.addOrderNStates(examples, new int[]{1}, null, null, null, null, false);
    crf.addStartState();
    crf.setWeightsDimensionAsIn(examples, true);

    if (initFrom != null) {
      crf.initializeApplicableParametersFrom(initFrom);
    }

    log.info("Starting syllchain training...");
    CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(crf, 8);
    trainer.setGaussianPriorVariance(2);
    trainer.setAddNoFactors(true);
//    trainer.setUseSomeUnsupportedTrick(true);
    trainer.train(examples);
    trainer.shutdown();
    watch.stop();

    log.info("SyllChain CRF Training took " + watch.toString());
    crf.getInputAlphabet().stopGrowth();
    crf.getOutputAlphabet().stopGrowth();
    return trainer;
  }
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
private TransducerTrainer trainOnce(Pipe pipe, InstanceList trainData) {
  Stopwatch watch = Stopwatch.createStarted();

  CRF crf = new CRF(pipe, null);

  // O,O    O,N   -O,C-
  // N,O    N,N   N,C
  // C,O    ?C,N?   C,C
  Pattern forbidden = null;
  if (USE_ONC_CODING) {
    forbidden = Pattern.compile("(O,C|<START>,C|O,<END>)", Pattern.CASE_INSENSITIVE);
  }
  crf.addOrderNStates(trainData, new int[]{1}, null, null, forbidden, null, false);
  crf.addStartState();
  crf.setWeightsDimensionAsIn(trainData);

  if (this.pullFrom != null) {
    crf.initializeApplicableParametersFrom(pullFrom);
  }

  log.info("Starting syll phone training...");
  CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(crf, 8);
  trainer.setGaussianPriorVariance(2);
  trainer.setAddNoFactors(false);
  trainer.setUseSomeUnsupportedTrick(true);
  trainer.train(trainData);
  trainer.shutdown();
  watch.stop();

  pipe.getAlphabet().stopGrowth();
  pipe.getTargetAlphabet().stopGrowth();
  log.info("Align Tag CRF Training took " + watch.toString());
  return trainer;
}
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
private static CRFTrainerByThreadedLabelLikelihood makeNewTrainer(CRF crf) {
  CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(crf, getCpuCount());
  trainer.setGaussianPriorVariance(2);
  trainer.setAddNoFactors(true);
  trainer.setUseSomeUnsupportedTrick(false);
  return trainer;
}
 

import cc.mallet.fst.CRFTrainerByThreadedLabelLikelihood; //导入依赖的package包/类
/** MultiSegmentationEvaluator */
    private static Fold evaluate(int iterationId, CRF crf,
            InstanceList trainingSet, InstanceList testingSet, int threads) {

        // TODO 1 see if it works (better) with simpler setup

        CRFTrainerByThreadedLabelLikelihood trainer = new CRFTrainerByThreadedLabelLikelihood(
                crf, threads);
        // CRFTrainerByLabelLikelihood trainer = new
        // CRFTrainerByLabelLikelihood(crf);
        trainer.setGaussianPriorVariance(1);

        String[] tags = new String[] { Jcas2TokenSequence.TARGET_I };
        String[] continueTags = tags;

        trainer.train(trainingSet);

        MyMultiSegmentationEvaluator eval = new MyMultiSegmentationEvaluator(
                new InstanceList[] { testingSet },//
                new String[] { "TTesting" }, tags, continueTags,
                PRINT_MISSCLASSIFIED);
        eval.evaluate(trainer); // eval at end of training

//        MultiSegmentationEvaluator evalOrig = new MultiSegmentationEvaluator(
//                new InstanceList[] { testingSet },//
//                new String[] { "TTesting" }, tags, continueTags);
//        evalOrig.evaluate(trainer); // eval at end of training

        LenientMultiSegmentationEvaluator evalLenient = new LenientMultiSegmentationEvaluator(
                new InstanceList[] { testingSet },//
                new String[] { "TTesting" }, tags, continueTags,
                PRINT_MISSCLASSIFIED);
        evalLenient.evaluate(trainer);

        LOG.info("FOLD {} --> " + eval + " lenient: {}", iterationId,
                evalLenient);
        return new Fold(eval);

        // TODO trainer.trainWithFeatureInduction

        // TODO
        // if ( runner.isInduceFeatures() ) {
        // // Number of maximizer iterations between each call to the Feature
        // Inducer. (10 in simpletagger and TUI)
        // int numIterationsBetweenFeatureInductions = 10;
        //
        // // Maximum number of rounds of feature induction. (20 in
        // simpleTagger, 99 in TUI)
        // int numFeatureInductions = 20;
        //
        // // Maximum number of features to induce at each round of induction.
        // (500 in simpletagger, 200 in TUI)
        // int numFeaturesPerFeatureInduction = 300;
        // // splits = new double[] {.1, .2, .5, .7}
        //
        // crft.trainWithFeatureInduction( training, null, testing, eval,
        // iterations,
        // numIterationsBetweenFeatureInductions, numFeatureInductions,
        // numFeaturesPerFeatureInduction, 0.5,
        // false, null );
        // } else {
        // // before
        // converged = crft.train( training ); // , iterations );
        // }
    }