Java TokenSequenceRemoveStopwords类代码示例

import cc.mallet.pipe.TokenSequenceRemoveStopwords; //导入依赖的package包/类
private static List<Pipe> getPipelist(Collection<String> stopwords) {
	// Begin by importing documents from text to feature sequences
	List<Pipe> pipeList = new ArrayList<Pipe>();

	// Pipes: lowercase, tokenize, remove stopwords, map to features
	pipeList.add(new CharSequenceLowercase());
	pipeList.add(new CharSequence2TokenSequence(Pattern.compile("\\p{L}[\\p{L}\\p{P}]+\\p{L}")));
	
	TokenSequenceRemoveStopwords stopwordsRemovalPipe = 
			new TokenSequenceRemoveStopwords(new File("mallet/stoplists/en.txt"), "UTF-8", false, false, false);
	if(!stopwords.isEmpty()) {
		stopwordsRemovalPipe.addStopWords(stopwords.toArray(new String[stopwords.size()]));
	}
	pipeList.add(stopwordsRemovalPipe);
	pipeList.add(new TokenSequence2FeatureSequence());
			
	return pipeList;
}
 

import cc.mallet.pipe.TokenSequenceRemoveStopwords; //导入依赖的package包/类
/**
 * Creates a list of Malelt instances from a list of documents
 * @param texts a list of documents
 * @return a list of Mallet instances
 * @throws IOException
 */
private InstanceList createInstanceList(List<String> texts) throws IOException
{
	ArrayList<Pipe> pipes = new ArrayList<Pipe>();
	pipes.add(new CharSequence2TokenSequence());
	pipes.add(new TokenSequenceLowercase());
	pipes.add(new TokenSequenceRemoveStopwords());
	pipes.add(new TokenSequence2FeatureSequence());
	InstanceList instanceList = new InstanceList(new SerialPipes(pipes));
	instanceList.addThruPipe(new ArrayIterator(texts));
	return instanceList;
}
 

import cc.mallet.pipe.TokenSequenceRemoveStopwords; //导入依赖的package包/类
/**
 * 
 * @param model
 * @param targetProcessing
 * @return
 */
private Pipe getPipe() {
	ArrayList<Pipe> pipes = new ArrayList<Pipe>();
	pipes.add(new Target2Label());
	pipes.add(new SaveDataInSource());
	pipes.add(new Input2CharSequence("UTF-8"));
	pipes.add(new CharSequence2TokenSequence(Pattern.compile("\\p{Alpha}+")));
	pipes.add(new TokenSequenceLowercase());
	pipes.add(new TokenSequenceRemoveStopwords(false, false));
	pipes.add(new TokenSequence2FeatureSequence());
	// pipes.add(new PrintInputAndTarget());
	return new SerialPipes(pipes);
}
 

import cc.mallet.pipe.TokenSequenceRemoveStopwords; //导入依赖的package包/类
public void testThree ()
{
	InstanceList il = new InstanceList (
		new SerialPipes(new Pipe[] {
			new Target2Label(),
			new CharSequence2TokenSequence(),
			new TokenSequenceLowercase(),
			new TokenSequenceRemoveStopwords(),
			new TokenSequence2FeatureSequence(),
			new FeatureSequence2FeatureVector()
		}));
	Iterator<Instance> pi = new FileIterator(new File("foo/bar"), null, Pattern.compile("^([^/]*)/"));
	il.addThruPipe (pi);
}
 

import cc.mallet.pipe.TokenSequenceRemoveStopwords; //导入依赖的package包/类
public Pipe buildPipe() {
    ArrayList pipeList = new ArrayList();

    // Read data from File objects
    pipeList.add(new Input2CharSequence("UTF-8"));

    // Regular expression for what constitutes a token.
    //  This pattern includes Unicode letters, Unicode numbers, 
    //   and the underscore character. Alternatives:
    //    "\\S+"   (anything not whitespace)
    //    "\\w+"    ( A-Z, a-z, 0-9, _ )
    //    "[\\p{L}\\p{N}_]+|[\\p{P}]+"   (a group of only letters and numbers OR
    //                                    a group of only punctuation marks)
    Pattern tokenPattern =
        Pattern.compile("[\\p{L}\\p{N}_]+");

    // Tokenize raw strings
    pipeList.add(new CharSequence2TokenSequence(tokenPattern));

    // Normalize all tokens to all lowercase
    pipeList.add(new TokenSequenceLowercase());

    // Remove stopwords from a standard English stoplist.
    //  options: [case sensitive] [mark deletions]
    pipeList.add(new TokenSequenceRemoveStopwords(false, false));

    // Rather than storing tokens as strings, convert 
    //  them to integers by looking them up in an alphabet.
    pipeList.add(new TokenSequence2FeatureSequence());

    // Do the same thing for the "target" field: 
    //  convert a class label string to a Label object,
    //  which has an index in a Label alphabet.
    pipeList.add(new Target2Label());

    // Now convert the sequence of features to a sparse vector,
    //  mapping feature IDs to counts.
    pipeList.add(new FeatureSequence2FeatureVector());

    // Print out the features and the label
    //pipeList.add(new PrintInputAndTarget());

    return new SerialPipes(pipeList);
}