Java TokenSequence.add方法代码示例

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	// xxx This doesn't seem so efficient.  Perhaps have TokenSequence
	// use a LinkedList, and remove Tokens from it? -?
	// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
	TokenSequence ret = new TokenSequence ();
	Token prevToken = null;
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		if (! stoplist.contains (caseSensitive ? t.getText() : t.getText().toLowerCase())) {
			// xxx Should we instead make and add a copy of the Token?
			ret.add (t);
			prevToken = t;
		} else if (markDeletions && prevToken != null)
			prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
	}
	carrier.setData(ret);
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {

    TokenSequence ts = (TokenSequence) carrier.getData();
    TokenSequence targetTokenSeq = new TokenSequence(ts.size());

    for (int i = 0; i < ts.size(); i++) {

        Token t = ts.get(i);
        // System.out.println(t.getText());
        String lineWithoutFirst = t.getText().replaceFirst("[^\\t]*\t", "");
        // System.out.println(lineWithoutFirst);
        // targetTokenSeq.add(lineSplit[0]);

        targetTokenSeq.add(t.getText().split("\t")[0]);
        t.setText(lineWithoutFirst);

    }
    carrier.setTarget(targetTokenSeq);
    carrier.setData(ts);

    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe(Instance carrier) {
	TokenSequence ts = (TokenSequence) carrier.getData();
	TokenSequence newTs = new TokenSequence();
	FeatureSequence labelSeq = new FeatureSequence(getTargetAlphabet());
	boolean lastWasSpace = true;
	StringBuffer sb = new StringBuffer();
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		if (t.getText().equals(" "))
			lastWasSpace = true;
		else {
			sb.append(t.getText());
			newTs.add(t);
			labelSeq.add(lastWasSpace ? "start" : "notstart");
			lastWasSpace = false;
		}
	}
	if (isTargetProcessing())
		carrier.setTarget(labelSeq);
	carrier.setData(newTs);
	carrier.setSource(sb.toString());
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	String newTerm = null;
	TokenSequence tmpTS = new TokenSequence();
	TokenSequence ts = (TokenSequence) carrier.getData();

	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		for(int j = 0; j < gramSizes.length; j++) {
			int len = gramSizes[j];
			if (len <= 0 || len > (i+1)) continue;
			if (len == 1) { tmpTS.add(t); continue; }
			newTerm = new String(t.getText());
			for(int k = 1; k < len; k++)
				newTerm = ts.get(i-k).getText() + "_" + newTerm;
			tmpTS.add(newTerm);
		}
	}

	carrier.setData(tmpTS);

	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public TokenSequence pipe (BufferedReader br)
	throws java.io.IOException
{
	final int BUFSIZE = 2048;
	char[] buf = new char[BUFSIZE];
	int count;
	StringBuffer sb = new StringBuffer (BUFSIZE);
	do {
		count = br.read (buf, 0, BUFSIZE);
		sb.append (buf);
	} while (count == BUFSIZE);
	lexer.setCharSequence ((CharSequence)sb);
	TokenSequence ts = new TokenSequence ();
	while (lexer.hasNext())
		ts.add (new Token ((String) lexer.next()));
	return ts;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	// xxx This doesn't seem so efficient.  Perhaps have TokenSequence
	// use a LinkedList, and remove Tokens from it? -?
	// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
	TokenSequence ret = new TokenSequence ();
	Token prevToken = null;
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		String s = t.getText();
		if (CharSequenceLexer.LEX_ALPHA.matcher(s).matches()) {
			ret.add (t);
			prevToken = t;
		}	else if (markDeletions && prevToken != null)
			prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
	}
	carrier.setData(ret);
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	TokenSequence targetTokenSeq = new TokenSequence (ts.size());
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		Matcher matcher = regex.matcher (t.getText());
		if (matcher.matches()) {
			targetTokenSeq.add (matcher.group(targetGroup));
			t.setText (matcher.group (dataGroup));
		} else {
			logger.warning ("Skipping token: No match of "+regex.pattern()
											+" at token #"+i+" with text "+t.getText());
		}
	}
	carrier.setTarget(targetTokenSeq);
	carrier.setData(ts);
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	long millis = System.currentTimeMillis();
	TokenSequence ts = (TokenSequence) carrier.getData();
	// xxx This doesn't seem so efficient.  Perhaps have TokenSequence
	// use a LinkedList, and remove Tokens from it? -?
	// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
	TokenSequence ret = new TokenSequence ();
	Token prevToken = null;
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		if (! stoplist.contains (caseSensitive ? t.getText() : t.getText().toLowerCase())) {
			// xxx Should we instead make and add a copy of the Token?
			ret.add (t);
			prevToken = t;
		} else if (markDeletions && prevToken != null)
			prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
	}
	carrier.setData(ret);
	//System.out.println(this.getClass().getCanonicalName() + "===" + (System.currentTimeMillis()-millis));
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier) {
	
	TokenSequence originalSequence = (TokenSequence) carrier.getData();
	TokenSequence newSequence = new TokenSequence();
	
	for (int i = 0; i < originalSequence.size(); i++) {
		Token t = originalSequence.get(i);
		
		boolean passed = true;
		String text = t.getText();
		for (Pattern pattern : stopPatterns) {
			Matcher matcher = pattern.matcher(text);
			if (matcher.matches()) {
				passed = false;
				break;
			}
		}
		
		if (passed) {
			newSequence.add (t);
		}
	}
	
	carrier.setData(newSequence);
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe(Instance carrier) {
    CharSequence string = (CharSequence) carrier.getData();
    TokenSequence ts = new StringTokenization(string);

    List<Term> terms = HanLP.segment(string.toString());

    for (Term term : terms) {
        if (term.word.length() < 2) {
            continue;
        }

        if (denydPosSet.contains(term.nature.name())) {
            continue;
        }

        ts.add(new Token(term.word));
    }

    carrier.setData(ts);
    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public TokenSequence pipe (BufferedReader br)
	throws IOException
{
	final int BUFSIZE = 2048;
	char[] buf = new char[BUFSIZE];
	int count;
	StringBuffer sb = new StringBuffer (BUFSIZE);
	do {
		count = br.read (buf, 0, BUFSIZE);
		sb.append (buf);
	} while (count == BUFSIZE);
	lexer.setCharSequence ((CharSequence)sb);
	TokenSequence ts = new TokenSequence ();
	while (lexer.hasNext())
		ts.add (new Token ((String) lexer.next()));
	return ts;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	if (carrier.getData() instanceof CharSequence)
		carrier.setData(new TokenSequence (ngramify ((CharSequence)carrier.getData())));
	else if (carrier.getData() instanceof TokenSequence) {
		TokenSequence ts = (TokenSequence) carrier.getData();
		TokenSequence ret = new TokenSequence ();
		for (int i = 0; i < ts.size(); i++)
			ret.add (ngramify (ts.get(i).getText()).toString());
		carrier.setData(ret);
	} else
		throw new IllegalArgumentException ("Unhandled type "+carrier.getData().getClass());
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	CharSequence string = (CharSequence) carrier.getData();
	lexer.setCharSequence (string);
	TokenSequence ts = new StringTokenization (string);
	while (lexer.hasNext()) {
		lexer.next();
		ts.add (new StringSpan (string, lexer.getStartOffset (), lexer.getEndOffset ()));
	}
	carrier.setData(ts);
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
	long millis=System.currentTimeMillis();
	CharSequence string = (CharSequence) carrier.getData();
	lexer.setCharSequence (string);
	TokenSequence ts = new StringTokenization (string);
	while (lexer.hasNext()) {
		lexer.next();
		ts.add (new StringSpan (string, lexer.getStartOffset (), lexer.getEndOffset ()));
	}
	
	carrier.setData(ts);
	//System.out.println(this.getClass().getCanonicalName() + "----" +(System.currentTimeMillis()-millis));
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
	String line = (String) carrier.getData();
	LabelAlphabet labels = (LabelAlphabet) getTargetAlphabet();
	TokenSequence ts = new TokenSequence();
	String[] tokens = line.split(" ");
	LabelSequence target = new LabelSequence(labels, tokens.length - 1);
	assert (tokens.length > 1);
	String prevTag = null;
	for (int i = 1; i < tokens.length; i++) {
		Matcher matcher = tokenPattern.matcher(tokens[i]);
		matcher.matches();
		String word = matcher.group(1);
		String tag = matcher.group(2);
		Token tok = new Token(word);
		tok.setFeatureValue(word, 1.0);
		ts.add(tok);
		String tmp = tag;
		if (targets.contains(tag)) {
			if (tag.equals(prevTag)) {
				tag = "I-GENE";
			} else {
				tag = "B-GENE";
			}
		} else {
			tag = "O";
		}
		target.add(tag);
		prevTag = tmp;
	}
	carrier.setData(ts);
	carrier.setTarget(target);
	return carrier;
}