Java FeatureVectorSequence类代码示例

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
/**
 * Runs constrained forward-backward. <p>
 *
 * If <tt>incrementor</tt> is null then do not update expectations due to
 * these computations. <p>
 *
 * The contribution of entropy to the expectations is multiplies by the
 * scaling factor.
 */
public EntropyLattice(FeatureVectorSequence fvs, double[][] gammas,
                      double[][][] xis, Transducer transducer,
                      Transducer.Incrementor incrementor,
                      double scalingFactor) {
  inputLength = fvs.size();
  latticeLength = inputLength + 1;
  this.transducer = transducer;
  numStates = transducer.numStates();

  nodes = new LatticeNode[latticeLength][numStates];

  // run forward-backward and compute the entropy
  entropy = this.forwardLattice(gammas, xis);
  double backwardEntropy = this.backwardLattice(gammas, xis);
  assert(Maths.almostEquals(entropy, backwardEntropy)) : entropy + " " + backwardEntropy;
   
  if (incrementor != null) {
    // add the entropy to expectations
    this.updateCounts(fvs, gammas, xis, scalingFactor, incrementor);
  }
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
public BitSet preProcess(InstanceList data) {
  // count
  int ii = 0;
  int fi;
  FeatureVector fv;
  BitSet bitSet = new BitSet(data.size());
  for (Instance instance : data) {
    FeatureVectorSequence fvs = (FeatureVectorSequence)instance.getData();
    for (int ip = 0; ip < fvs.size(); ip++) {
      fv = fvs.get(ip);
      for (int loc = 0; loc < fv.numLocations(); loc++) {
        fi = fv.indexAtLocation(loc);
        if (constraints.containsKey(fi)) {
          constraints.get(fi).count += 1;
          bitSet.set(ii);
        }
      }
    }
    ii++;
  }
  return bitSet;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
/**
 * Resets, computes and fills expectations from all instances, also updating
 * the entropy value. <p>
 *
 * Analogous to <tt>CRFOptimizableByLabelLikelihood.getExpectationValue<tt>.
 */
public void computeExpectations() {
	expectations.zero();

  // now, update the expectations due to each instance for entropy reg.
  for (int ii = 0; ii < data.size(); ii++) {
    FeatureVectorSequence input = (FeatureVectorSequence) data.get(ii).getData();
    SumLattice lattice = new SumLatticeDefault(crf,input, true);

    // udpate the expectations
    EntropyLattice entropyLattice = new EntropyLattice(
        input, lattice.getGammas(), lattice.getXis(), crf,
        incrementor, scalingFactor);
    cachedValue += entropyLattice.getEntropy();
  }
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
public BitSet preProcess(InstanceList data) {
  // count
  BitSet bitSet = new BitSet(data.size());
  int ii = 0;
  for (Instance instance : data) {
    FeatureVectorSequence fvs = (FeatureVectorSequence)instance.getData();
    for (int ip = 1; ip < fvs.size(); ip++) {
      for (int fi : constraintsMap.keys()) {
        // binary constraint features
        if (fvs.get(ip).location(fi) >= 0) {
          constraintsList.get(constraintsMap.get(fi)).count += 1;
          bitSet.set(ii);
        }
      }
    }
    ii++;
  }
  return bitSet;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
private double computeLikelihood(InstanceList trainingSample) {
	double loglik = 0.0;
	for (int i = 0; i < trainingSample.size(); i++) {
		Instance trainingInstance = trainingSample.get(i);
		FeatureVectorSequence fvs = (FeatureVectorSequence) trainingInstance
				.getData();
		Sequence labelSequence = (Sequence) trainingInstance.getTarget();
		loglik += new SumLatticeDefault(crf, fvs, labelSequence, null)
				.getTotalWeight();
		loglik -= new SumLatticeDefault(crf, fvs, null, null)
				.getTotalWeight();
	}
	constraints.zero();
	expectations.zero();
	return loglik;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
/**
 * Adjust the parameters by learning rate according to the gradient of this
 * single Instance, and return the true label sequence likelihood.
 */
public double trainIncrementalLikelihood(Instance trainingInstance,
		double rate) {
	double singleLoglik;
	constraints.zero();
	expectations.zero();
	FeatureVectorSequence fvs = (FeatureVectorSequence) trainingInstance
			.getData();
	Sequence labelSequence = (Sequence) trainingInstance.getTarget();
	singleLoglik = new SumLatticeDefault(crf, fvs, labelSequence,
			constraints.new Incrementor()).getTotalWeight();
	singleLoglik -= new SumLatticeDefault(crf, fvs, null,
			expectations.new Incrementor()).getTotalWeight();
	// Calculate parameter gradient given these instances: (constraints -
	// expectations)
	constraints.plusEquals(expectations, -1);
	// Change the parameters a little by this difference, obeying
	// weightsFrozen
	crf.parameters.plusEquals(constraints, rate, true);

	return singleLoglik;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
/**
 * Set the constraints by running forward-backward with the <i>output label
 * sequence provided</i>, thus restricting it to only those paths that agree with
 * the label sequence.
 */
protected void gatherConstraints(InstanceList ilist) {
	logger.info("Gathering constraints...");
	assert (constraints.structureMatches(crf.parameters));
	constraints.zero();

	for (Instance instance : ilist) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = ilist.getInstanceWeight(instance);
		Transducer.Incrementor incrementor =
			instanceWeight == 1.0 ? constraints.new Incrementor()
		: constraints.new WeightedIncrementor(instanceWeight);
			new SumLatticeDefault (this.crf, input, output, incrementor); 
	}
	constraints.assertNotNaNOrInfinite();
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
protected void gatherConstraints (InstanceList ilist)
	{
		// Set the constraints by running forward-backward with the *output
		// label sequence provided*, thus restricting it to only those
		// paths that agree with the label sequence.
		// Zero the constraints[]
		// Reset constraints[] to zero before we fill them again
		assert (constraints.structureMatches(crf.parameters));
		constraints.zero();

		for (Instance instance : ilist) {
			FeatureVectorSequence input = (FeatureVectorSequence) instance.getData();
			FeatureSequence output = (FeatureSequence) instance.getTarget();
			double instanceWeight = ilist.getInstanceWeight(instance);
			//System.out.println ("Constraint-gathering on instance "+i+" of "+ilist.size());
			Transducer.Incrementor incrementor = instanceWeight == 1.0 ? constraints.new Incrementor() : constraints.new WeightedIncrementor(instanceWeight);
			new SumLatticeDefault (this.crf, input, output, incrementor); 
		}
//		System.out.println ("testing Value and Gradient");
//		TestOptimizable.testValueAndGradientCurrentParameters (this);
	}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
protected void gatherConstraints() {
	// Set the constraints by running forward-backward with the *output
	// label sequence provided*, thus restricting it to only those
	// paths that agree with the label sequence.
	// Zero the constraints[]
	// Reset constraints[] to zero before we fill them again
	assert (constraints.structureMatches(crf.parameters));
	constraints.zero();

	for (Instance instance : sourceInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = sourceInstances.getInstanceWeight(instance);
		// System.out.println
		// ("Constraint-gathering on instance "+i+" of "+ilist.size());
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? constraints.new Incrementor()
				: constraints.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, output, incrementor);
	}
	// System.out.println ("testing Value and Gradient");
	// TestOptimizable.testValueAndGradientCurrentParameters (this);
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
protected void getSourceExpectations() {
	assert (sourceExpectations.structureMatches(crf.parameters));
	sourceExpectations.zero();

	for (Instance instance : sourceInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = sourceInstances.getInstanceWeight(instance);
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? sourceExpectations.new Incrementor()
				: sourceExpectations.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, output, incrementor);
	}
	double factor = 1.0 / sourceInstances.size();
	for (int i = 0; i < sourceExpectations.weights.length; i++) {
		log(sourceExpectations.weights[i], factor);
	}
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
protected void getTargetExpectations() {
	// Reset expectations to zero before we fill them again
	assert (targetExpectations.structureMatches(crf.parameters));
	targetExpectations.zero();

	// Calculate the value of each instance, and also fill in expectations
	for (Instance instance : targetInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		double instanceWeight = targetInstances.getInstanceWeight(instance);
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? targetExpectations.new Incrementor()
				: targetExpectations.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, null, incrementor);
	}
	double factor = 1.0 / targetInstances.size();
	for (int i = 0; i < targetExpectations.weights.length; i++) {
		log(targetExpectations.weights[i], factor);
	}
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
public Sample getSample(int index) {
	Instance2Sample s = integerToSampleMap.get(index);
	if (s == null) {
		int offset = 0;
		Iterator<?> it = instanceList.iterator();
		while (it.hasNext()) {
			Instance instance = (Instance) it.next();
			Object data = instance.getData();
			if (data instanceof FeatureVectorSequence) {
				FeatureVectorSequence fvs = (FeatureVectorSequence) data;
				int size = fvs.size();
				if (index - offset < size) {
					s = new Instance2Sample(instance, index - offset);
					integerToSampleMap.put(index, s);
					return s;
				} else {
					offset += fvs.size();
				}
			}
		}
	}
	return s;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
/**
 * 
 * @param num_best
 * @return
 */
@SuppressWarnings("unchecked")
public List<CRFResult> test(Integer num_best) {
	List<CRFResult> groups = new ArrayList<CRFResult>();
	for (Instance instance : this.test_data) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance.getData();
		List<Sequence<Object>> seqs = null;
		if (num_best > 1)
			seqs = new MaxLatticeDefault(model, input, null, 100000).bestOutputSequences(num_best);
		else
			seqs = Arrays.asList((Sequence<Object>) model.transduce(input));
		if (isError(seqs, input))
			System.err.println("[ERROR] Error output at " + input);

		List<List<String>> outputs = new ArrayList<List<String>>();
		for (Sequence<Object> seq : seqs)
			outputs.add(toTagStrings(seq));

		groups.add(new CRFResult(input, outputs));
	}

	return groups;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance inst) {
  PartialTagging data = (PartialTagging) inst.getData();

  List<FeatureVector> vectors = Lists.newArrayList();
  for (Integer index : data.getPredictionIndexes()) {
    processPipes(data, index);
    vectors.add(makeVector(data, index));
  }
  if (vectors.isEmpty()) {
    throw new IllegalArgumentException("Cant seqvow a word with no vowels");
  }
  updateTarget(data, inst);
  inst.setData(new FeatureVectorSequence(vectors.toArray(new FeatureVector[vectors.size()])));
  return inst;
}
 

import cc.mallet.types.FeatureVectorSequence; //导入依赖的package包/类
private static RankedFeatureVector countFeatures(InstanceList ilist, boolean countInstances) {
  int numFeatures = ilist.getDataAlphabet().size();
  double[] counts = new double[numFeatures];
  for (int i = 0; i < ilist.size(); i++) {
    Instance inst = ilist.get(i);
    if (ilist.getInstanceWeight(i) == 0) {
      continue;
    }
    Object data = inst.getData();
    if (data instanceof FeatureVectorSequence) {
      FeatureVectorSequence fvs = (FeatureVectorSequence) data;
      for (int j = 0; j < fvs.size(); j++) {
        countVector(counts, fvs.get(j), countInstances);
      }
    } else {
      throw new IllegalArgumentException("Currently only handles FeatureVectorSequence data");
    }
  }
  return new RankedFeatureVector(ilist.getDataAlphabet(), counts);
}