Java Transducer类代码示例

import cc.mallet.fst.Transducer; //导入依赖的package包/类
public Transducer trainModel(InstanceList trainingData, String options) {

    TransducerTrainer trainer = createTrainer(trainingData, info, options);
    Parms parms = new Parms(options,"i:iterations:i","V:verbose:b");
    boolean verbose = (boolean)parms.getValueOrElse("verbose", false);
    int iters = (int) parms.getValueOrElse("iterations", 0);
    if(iters==0) iters = Integer.MAX_VALUE;
    try {
      trainer.train(trainingData, iters);
    } catch(OptimizationException ex) {
      System.err.println("Encountered an OptimizationException during training (CONTINUING!): "+ex.getMessage());
      ex.printStackTrace(System.err);
      System.err.println("We ignore this exception and try to use the model so far ...");
    }
    if(verbose) 
      trainer.getTransducer().print();
    Transducer td = trainer.getTransducer();
    return td;
  }
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
/**
 *
 * @param n:
 *            positive integer. Creates all possible orders from 0 until and
 *            including n
 */
public void addOrderNStates(int n, InstanceList trainingInstances) {

    Pattern forbiddenPat = Pattern.compile("\\s");
    Pattern allowedPat = Pattern.compile(".*");
    List<Integer> orders = new ArrayList<Integer>();
    for (int i = 0; i <= n; i++) {
        orders.add(i);
    }
    int[] ordersArray = null;
    if (orders.size() > 0) {
        ordersArray = ArrayUtils.toPrimitive(orders.toArray(new Integer[orders.size()]));
    }
    String startName = this.crf.addOrderNStates(trainingInstances, ordersArray, null, "O", forbiddenPat, allowedPat,
            true);
    for (int i = 0; i < this.crf.numStates(); i++) {
        this.crf.getState(i).setInitialWeight(Transducer.IMPOSSIBLE_WEIGHT);
    }
    this.crf.getState(startName).setInitialWeight(0.0);
    this.crf.setWeightsDimensionDensely();
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
/**
 * Runs constrained forward-backward. <p>
 *
 * If <tt>incrementor</tt> is null then do not update expectations due to
 * these computations. <p>
 *
 * The contribution of entropy to the expectations is multiplies by the
 * scaling factor.
 */
public EntropyLattice(FeatureVectorSequence fvs, double[][] gammas,
                      double[][][] xis, Transducer transducer,
                      Transducer.Incrementor incrementor,
                      double scalingFactor) {
  inputLength = fvs.size();
  latticeLength = inputLength + 1;
  this.transducer = transducer;
  numStates = transducer.numStates();

  nodes = new LatticeNode[latticeLength][numStates];

  // run forward-backward and compute the entropy
  entropy = this.forwardLattice(gammas, xis);
  double backwardEntropy = this.backwardLattice(gammas, xis);
  assert(Maths.almostEquals(entropy, backwardEntropy)) : entropy + " " + backwardEntropy;
   
  if (incrementor != null) {
    // add the entropy to expectations
    this.updateCounts(fvs, gammas, xis, scalingFactor, incrementor);
  }
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
/**
 * Apply a transducer to an input sequence to produce the k highest-scoring
 * output sequences.
 *
 * @param model the <code>Transducer</code>
 * @param input the input sequence
 * @param k the number of answers to return
 * @return array of the k highest-scoring output sequences
 */
public static Sequence[] apply(Transducer model, Sequence input, int k)
{
  Sequence[] answers;
  if (k == 1) {
    answers = new Sequence[1];
    answers[0] = model.transduce (input);
  }
  else {
    MaxLatticeDefault lattice =
            new MaxLatticeDefault (model, input, null, cacheSizeOption.value());

    answers = lattice.bestOutputSequences(k).toArray(new Sequence[0]);
  }
  return answers;
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
private static void outputLatticeRows (PrintWriter out, MaxLattice lattice, int start, int end)
{
  DecimalFormat f = new DecimalFormat ("0.##");
  Transducer ducer = lattice.getTransducer ();
  int max = Math.min (numMaxViterbi, ducer.numStates());
  List<Sequence<Transducer.State>> stateSequences = lattice.bestStateSequences(max);
  for (int k = 0; k < max; k++) {
    out.println ("  <tr class=\"delta\">");
    out.println ("    <td class=\"label\">&delta; rank "+k+"</td>");
    for (int ip = start; ip < end; ip++) {
        Transducer.State state = stateSequences.get(k).get(ip+1);
      if (state.getName().equals (lattice.bestOutputSequence().get(ip))) {
        out.print ("<td class=\"viterbi\">");
      } else {
        out.print ("<td>");
      }
      out.print (state.getName()+"<br />"+f.format (-lattice.getDelta (ip+1, state.getIndex ()))+"</td>");
    }
    out.println ("</tr>");
  }
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
protected void gatherConstraints() {
	// Set the constraints by running forward-backward with the *output
	// label sequence provided*, thus restricting it to only those
	// paths that agree with the label sequence.
	// Zero the constraints[]
	// Reset constraints[] to zero before we fill them again
	assert (constraints.structureMatches(crf.parameters));
	constraints.zero();

	for (Instance instance : sourceInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = sourceInstances.getInstanceWeight(instance);
		// System.out.println
		// ("Constraint-gathering on instance "+i+" of "+ilist.size());
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? constraints.new Incrementor()
				: constraints.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, output, incrementor);
	}
	// System.out.println ("testing Value and Gradient");
	// TestOptimizable.testValueAndGradientCurrentParameters (this);
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
protected void getSourceExpectations() {
	assert (sourceExpectations.structureMatches(crf.parameters));
	sourceExpectations.zero();

	for (Instance instance : sourceInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = sourceInstances.getInstanceWeight(instance);
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? sourceExpectations.new Incrementor()
				: sourceExpectations.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, output, incrementor);
	}
	double factor = 1.0 / sourceInstances.size();
	for (int i = 0; i < sourceExpectations.weights.length; i++) {
		log(sourceExpectations.weights[i], factor);
	}
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
protected void getTargetExpectations() {
	// Reset expectations to zero before we fill them again
	assert (targetExpectations.structureMatches(crf.parameters));
	targetExpectations.zero();

	// Calculate the value of each instance, and also fill in expectations
	for (Instance instance : targetInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		double instanceWeight = targetInstances.getInstanceWeight(instance);
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? targetExpectations.new Incrementor()
				: targetExpectations.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, null, incrementor);
	}
	double factor = 1.0 / targetInstances.size();
	for (int i = 0; i < targetExpectations.weights.length; i++) {
		log(targetExpectations.weights[i], factor);
	}
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
/**
 * 
 * @param num_iterations
 * @return
 */
public Boolean train(Integer num_iterations) {
	this.model = new CRF(this.train_data.getPipe(), (Pipe) null);
	for (int i = 0; i < this.model.numStates(); i++)
		this.model.getState(i).setInitialWeight(Transducer.IMPOSSIBLE_WEIGHT);
	String startName = this.model.addOrderNStates(this.train_data, new int[] { 1 }, null, DEFAULT_LABEL, Pattern.compile("\\s"), Pattern.compile(".*"), true);
	this.model.getState(startName).setInitialWeight(0.0);

	CRFTrainerByLabelLikelihood crft = new CRFTrainerByLabelLikelihood(this.model);
	crft.setGaussianPriorVariance(DEFAULT_PRIOR_VARIANCE);
	crft.setUseSparseWeights(true);
	crft.setUseSomeUnsupportedTrick(true);

	for (int i = 0; i < num_iterations; i++)
		if (crft.train(this.train_data, 1))
			break;

	return this.model != null;
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
private static void configure(CRF _crf, InstanceList trainingSet) {

        // crf.addStatesForLabelsConnectedAsIn(trainingSet);
        // CRFTrainerByLabelLikelihood trainer = new
        // CRFTrainerByLabelLikelihood(
        // crf);
        // trainer.setGaussianPriorVariance(1d);

        int[] orders = new int[] { 1 };
        Pattern forbiddenPat = Pattern.compile("\\s");
        Pattern allowedPat = Pattern.compile(".*");

        String outside = Jcas2TokenSequence.TARGET_O;
        String startName = _crf.addOrderNStates(trainingSet, orders, null,
                outside, forbiddenPat, allowedPat, true);
        // String startName = crf.addOrderNStates(trainingSet, orders, null,
        // null, null, null, true);

        for (int i = 0; i < _crf.numStates(); i++)
            _crf.getState(i).setInitialWeight(Transducer.IMPOSSIBLE_WEIGHT);
        _crf.getState(startName).setInitialWeight(0.0);
    }
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
public Double call() throws Exception {
	double value = 0;
	
	for (int ii = start; ii < end; ii++) {
		Instance inst = trainingSet.get(ii);
		Sequence input = (Sequence) inst.getData();
		double initProbs[] = initialProbList.get(ii);
		double finalProbs[] = finalProbList.get(ii);
		double transProbs[][][] = transitionProbList.get(ii);

     double[][][] cachedDots = new double[input.size()][crf.numStates()][crf.numStates()];
     for (int j = 0; j < input.size(); j++) {
       for (int k = 0; k < crf.numStates(); k++) {
         for (int l = 0; l < crf.numStates(); l++) {
           cachedDots[j][k][l] = Transducer.IMPOSSIBLE_WEIGHT;
         }
       }
     }
		
		double labeledWeight = new SumLatticeKL(crf, input, initProbs, 
		    finalProbs, transProbs, cachedDots, null).getTotalWeight();
		value += labeledWeight;

		//double unlabeledWeight = new SumLatticeDefault(crf, input, 
		//    expectationsCopy.new Incrementor()).getTotalWeight();
      double unlabeledWeight = new SumLatticeDefaultCachedDot(crf, input, null, 
          cachedDots, expectationsCopy.new Incrementor(), false, null).getTotalWeight();

		value -= unlabeledWeight;
	}
	return value;
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
protected CachedDotTransitionIterator(State source, Object fv,
    String output, double[] dots) {
  this.source = source;
  this.input = fv;
  this.weights = new double[source.numDestinations()];
  for (int i = 0; i < source.numDestinations(); i++) {
    weights[i] = dots[source.getDestinationState(i).getIndex()];
  }
  // Prepare nextIndex, pointing at the next non-impossible transition
  nextIndex = 0;
  while (nextIndex < source.numDestinations()
      && weights[nextIndex] == Transducer.IMPOSSIBLE_WEIGHT)
    nextIndex++;
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
public Transducer.State nextState() {
  assert (nextIndex < source.numDestinations());
  index = nextIndex;
  nextIndex++;
  while (nextIndex < source.numDestinations()
      && weights[nextIndex] == Transducer.IMPOSSIBLE_WEIGHT)
    nextIndex++;
  return source.getDestinationState(index);
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
public static CRF getCRF(InstanceList training, int[] orders, String defaultLabel, String forbidden, String allowed, boolean connected) { 
  Pattern forbiddenPat = Pattern.compile(forbidden);
  Pattern allowedPat = Pattern.compile(allowed);
  CRF crf = new CRF(training.getPipe(), (Pipe)null);
  String startName = crf.addOrderNStates(training, orders, null,
      defaultLabel, forbiddenPat, allowedPat, connected);
  for (int i = 0; i < crf.numStates(); i++)
    crf.getState(i).setInitialWeight (Transducer.IMPOSSIBLE_WEIGHT);
  crf.getState(startName).setInitialWeight(0.0);
  crf.setWeightsDimensionDensely();
  return crf;
}
 

import cc.mallet.fst.Transducer; //导入依赖的package包/类
public LatticeNode() {
  alpha = new LogNumber[numStates];
  beta = new LogNumber[numStates];
  for (int si = 0; si < numStates; ++si) {
    alpha[si] = new LogNumber(Transducer.IMPOSSIBLE_WEIGHT,true);
    beta[si] =  new LogNumber(Transducer.IMPOSSIBLE_WEIGHT,true);
  }
}