Java FeatureSequence类代码示例

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
public void printState (ArrayList<Topication> dataset, PrintStream out) {

		out.println ("#doc source pos typeindex type topic");

		for (int di = 0; di < dataset.size(); di++) {
			FeatureSequence tokenSequence =	(FeatureSequence) dataset.get(di).instance.getData();
			LabelSequence topicSequence =	dataset.get(di).topicSequence;

			String source = "NA";
			if (dataset.get(di).instance.getSource() != null) {
				source = dataset.get(di).instance.getSource().toString();
			}

			for (int pi = 0; pi < topicSequence.getLength(); pi++) {
				int type = tokenSequence.getIndexAtPosition(pi);
				int topic = topicSequence.getIndexAtPosition(pi);
				out.print(di); out.print(' ');
				out.print(source); out.print(' ');
				out.print(pi); out.print(' ');
				out.print(type); out.print(' ');
				out.print(alphabet.lookupObject(type)); out.print(' ');
				out.print(topic); out.println();
			}
		}
	}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
private boolean[][] labelConnectionsIn (InstanceList trainingSet, String start)
{
	int numLabels = outputAlphabet.size();
	boolean[][] connections = new boolean[numLabels][numLabels];
	for (int i = 0; i < trainingSet.size(); i++) {
		Instance instance = trainingSet.get(i);
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		for (int j = 1; j < output.size(); j++) {
			int sourceIndex = outputAlphabet.lookupIndex (output.get(j-1));
			int destIndex = outputAlphabet.lookupIndex (output.get(j));
			assert (sourceIndex >= 0 && destIndex >= 0);
			connections[sourceIndex][destIndex] = true;
		}
	}

	// Handle start state
	if (start != null) {
		int startIndex = outputAlphabet.lookupIndex (start);
		for (int j = 0; j < outputAlphabet.size(); j++) {
			connections[startIndex][j] = true;
		}
	}

	return connections;
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
/**
 * Set the constraints by running forward-backward with the <i>output label
 * sequence provided</i>, thus restricting it to only those paths that agree with
 * the label sequence.
 */
protected void gatherConstraints(InstanceList ilist) {
	logger.info("Gathering constraints...");
	assert (constraints.structureMatches(crf.parameters));
	constraints.zero();

	for (Instance instance : ilist) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = ilist.getInstanceWeight(instance);
		Transducer.Incrementor incrementor =
			instanceWeight == 1.0 ? constraints.new Incrementor()
		: constraints.new WeightedIncrementor(instanceWeight);
			new SumLatticeDefault (this.crf, input, output, incrementor); 
	}
	constraints.assertNotNaNOrInfinite();
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
public Transducer.TransitionIterator transitionIterator(
		Sequence inputSequence, int inputPosition,
		Sequence outputSequence, int outputPosition) {
	if (inputPosition < 0 || outputPosition < 0)
		throw new UnsupportedOperationException(
				"Epsilon transitions not implemented.");
	if (inputSequence == null)
		throw new UnsupportedOperationException(
				"HMMs are generative models; but this is not yet implemented.");
	if (!(inputSequence instanceof FeatureSequence))
		throw new UnsupportedOperationException(
				"HMMs currently expect Instances to have FeatureSequence data");
	return new TransitionIterator(this,
			(FeatureSequence) inputSequence, inputPosition,
			(outputSequence == null ? null : (String) outputSequence
					.get(outputPosition)), hmm);
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
private void readObject(ObjectInputStream in) throws IOException,
		ClassNotFoundException {
	int version = in.readInt();
	source = (State) in.readObject();
	index = in.readInt();
	nextIndex = in.readInt();
	inputPos = in.readInt();
	int size = in.readInt();
	if (size == NULL_INTEGER) {
		weights = null;
	} else {
		weights = new double[size];
		for (int i = 0; i < size; i++) {
			weights[i] = in.readDouble();
		}
	}
	inputSequence = (FeatureSequence) in.readObject();
	inputFeature = (Integer) in.readObject();
	hmm = (HMM) in.readObject();
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
public Instance pipe(Instance carrier) {
	TokenSequence ts = (TokenSequence) carrier.getData();
	TokenSequence newTs = new TokenSequence();
	FeatureSequence labelSeq = new FeatureSequence(getTargetAlphabet());
	boolean lastWasSpace = true;
	StringBuffer sb = new StringBuffer();
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		if (t.getText().equals(" "))
			lastWasSpace = true;
		else {
			sb.append(t.getText());
			newTs.add(t);
			labelSeq.add(lastWasSpace ? "start" : "notstart");
			lastWasSpace = false;
		}
	}
	if (isTargetProcessing())
		carrier.setTarget(labelSeq);
	carrier.setData(newTs);
	carrier.setSource(sb.toString());
	return carrier;
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
protected void gatherConstraints (InstanceList ilist)
	{
		// Set the constraints by running forward-backward with the *output
		// label sequence provided*, thus restricting it to only those
		// paths that agree with the label sequence.
		// Zero the constraints[]
		// Reset constraints[] to zero before we fill them again
		assert (constraints.structureMatches(crf.parameters));
		constraints.zero();

		for (Instance instance : ilist) {
			FeatureVectorSequence input = (FeatureVectorSequence) instance.getData();
			FeatureSequence output = (FeatureSequence) instance.getTarget();
			double instanceWeight = ilist.getInstanceWeight(instance);
			//System.out.println ("Constraint-gathering on instance "+i+" of "+ilist.size());
			Transducer.Incrementor incrementor = instanceWeight == 1.0 ? constraints.new Incrementor() : constraints.new WeightedIncrementor(instanceWeight);
			new SumLatticeDefault (this.crf, input, output, incrementor); 
		}
//		System.out.println ("testing Value and Gradient");
//		TestOptimizable.testValueAndGradientCurrentParameters (this);
	}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
public void testNewPutSizeFreeze ()
{
	Alphabet dict = new Alphabet ();
	FeatureSequence fs = new FeatureSequence (dict, 10);
	fs.add (dict.lookupIndex ("apple"));
	fs.add (dict.lookupIndex ("bear"));
	fs.add (dict.lookupIndex ("car"));
	fs.add (dict.lookupIndex ("door"));
	assertTrue (fs.size() == 4);
	double[] weights = new double[4];
	fs.addFeatureWeightsTo (weights);
	assertTrue (weights[1] == 1.0);

	fs.add (dict.lookupIndex ("bear"));
	int[] feats = fs.toFeatureIndexSequence();
	assertTrue (feats[0] == 0);
	assertTrue (feats[1] == 1);
	assertTrue (feats[2] == 2);
	assertTrue (feats[3] == 3);
	assertTrue (feats[4] == 1);
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
protected void setUp ()
{
	dict = new Alphabet ();
	fs = new FeatureSequence (dict, 2);
	fs.add (dict.lookupIndex ("a"));
	fs.add (dict.lookupIndex ("n"));
	fs.add (dict.lookupIndex ("d"));
	fs.add (dict.lookupIndex ("r"));
	fs.add (dict.lookupIndex ("e"));
	fs.add (dict.lookupIndex ("w"));
	fs.add (dict.lookupIndex ("m"));
	fs.add (dict.lookupIndex ("c"));
	fs.add (dict.lookupIndex ("c"));
	fs.add (dict.lookupIndex ("a"));
	fs.add (dict.lookupIndex ("l"));
	fs.add (dict.lookupIndex ("l"));
	fs.add (dict.lookupIndex ("u"));
	fs.add (dict.lookupIndex ("m"));
	//System.out.println (fs.toString());
	fv = new FeatureVector (fs);
	//System.out.println (fs.toString());
	//System.out.println (fv.toString());
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
public Instance pipe (Instance carrier) {

		long start = System.nanoTime();

		try {
			ArrayList<String> tokens = (ArrayList<String>) carrier.getData();
			FeatureSequence featureSequence =
				new FeatureSequence ((Alphabet) getDataAlphabet(), tokens.size());
			for (int i = 0; i < tokens.size(); i++) {
				featureSequence.add (tokens.get(i));
			}
			carrier.setData(featureSequence);
			
			totalNanos += System.nanoTime() - start;
		} catch (ClassCastException cce) {
			System.err.println("Expecting ArrayList<String>, found " + carrier.getData().getClass());
		}

		return carrier;
	}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
protected void gatherConstraints() {
	// Set the constraints by running forward-backward with the *output
	// label sequence provided*, thus restricting it to only those
	// paths that agree with the label sequence.
	// Zero the constraints[]
	// Reset constraints[] to zero before we fill them again
	assert (constraints.structureMatches(crf.parameters));
	constraints.zero();

	for (Instance instance : sourceInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = sourceInstances.getInstanceWeight(instance);
		// System.out.println
		// ("Constraint-gathering on instance "+i+" of "+ilist.size());
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? constraints.new Incrementor()
				: constraints.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, output, incrementor);
	}
	// System.out.println ("testing Value and Gradient");
	// TestOptimizable.testValueAndGradientCurrentParameters (this);
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
protected void getSourceExpectations() {
	assert (sourceExpectations.structureMatches(crf.parameters));
	sourceExpectations.zero();

	for (Instance instance : sourceInstances) {
		FeatureVectorSequence input = (FeatureVectorSequence) instance
				.getData();
		FeatureSequence output = (FeatureSequence) instance.getTarget();
		double instanceWeight = sourceInstances.getInstanceWeight(instance);
		Transducer.Incrementor incrementor = instanceWeight == 1.0 ? sourceExpectations.new Incrementor()
				: sourceExpectations.new WeightedIncrementor(instanceWeight);
		new SumLatticeDefault(this.crf, input, output, incrementor);
	}
	double factor = 1.0 / sourceInstances.size();
	for (int i = 0; i < sourceExpectations.weights.length; i++) {
		log(sourceExpectations.weights[i], factor);
	}
}
 

import cc.mallet.types.FeatureSequence; //导入依赖的package包/类
public static double getIdfFromType(int type,InstanceList instances) {
    double idf = 0.0;
    int df = 0;

    for (int i = 0; i < instances.size(); i++) {
        FeatureSequence docFeatures = (FeatureSequence) instances.get(i).getData();
        int[] features = docFeatures.getFeatures();
        for(int f:features) {
            if(type==f){
                df++;
                break;
            }
        }
    }
    if(!(df==0)&&!(df==instances.size())){
        idf=Math.log((double) (instances.size())/df);
        idf=idf/Math.log(instances.size());
    }
    return idf;
}