Java TokenSequence类代码示例

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	// xxx This doesn't seem so efficient.  Perhaps have TokenSequence
	// use a LinkedList, and remove Tokens from it? -?
	// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
	TokenSequence ret = new TokenSequence ();
	Token prevToken = null;
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		if (! stoplist.contains (caseSensitive ? t.getText() : t.getText().toLowerCase())) {
			// xxx Should we instead make and add a copy of the Token?
			ret.add (t);
			prevToken = t;
		} else if (markDeletions && prevToken != null)
			prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
	}
	carrier.setData(ret);
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance carrier) {

    TokenSequence ts = (TokenSequence) carrier.getData();
    for (int i = 0; i < ts.size(); i++) {
        Token t = ts.get(i);
        int splitLength = t.getText().split("\t").length;
        if (splitLength == this.minLineLength) {
            t.setText("O\t" + t.getText());
        } else {
            if (splitLength != (this.minLineLength + 1)) {
                System.err.println("input line does not have length " + this.minLineLength + " or "
                        + (this.minLineLength + 1) + " but " + splitLength + ": " + t.getText());
            }
        }
    }
    carrier.setData(ts);

    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
    TokenSequence tokenSequence = (TokenSequence) carrier.getData();
    for (int i = 0; i < tokenSequence.size(); i++) {
        Token token = tokenSequence.get(i);
        String tokenText = token.getText().split(this.csvSeparator)[0];
        int count = 0;
        Matcher matcher = this.pattern.matcher(tokenText);
        while (matcher.find()) {
            count++;
        }
        // int count = StringUtils.countMatches(tokenText, this.subString);
        if (count > 0) {
            // token.setFeatureValue(this.feature + "=" + count, 1.0);
            token.setFeatureValue(this.feature, count);
        }
    }
    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance instance) {

    TokenSequence sequence = (TokenSequence) instance.getData();

    for (Token token : sequence) {
        String[] tokenSplit = token.getText().trim().split("\\s");
        for (int i = 0; i < tokenSplit.length; i++) {
            String normalizedTokenPart = tokenSplit[i].replaceAll("[^\\p{L}]", "");
            if ((normalizedTokenPart.length() > 0) && this.nameCountMap.containsKey((normalizedTokenPart))) {
                token.setFeatureValue(this.featureName + "@" + i, 1.0);
            }
        }
    }
    return instance;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance carrier) {

    TokenSequence ts = (TokenSequence) carrier.getData();
    TokenSequence targetTokenSeq = new TokenSequence(ts.size());

    for (int i = 0; i < ts.size(); i++) {

        Token t = ts.get(i);
        // System.out.println(t.getText());
        String lineWithoutFirst = t.getText().replaceFirst("[^\\t]*\t", "");
        // System.out.println(lineWithoutFirst);
        // targetTokenSeq.add(lineSplit[0]);

        targetTokenSeq.add(t.getText().split("\t")[0]);
        t.setText(lineWithoutFirst);

    }
    carrier.setTarget(targetTokenSeq);
    carrier.setData(ts);

    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
    this.referenceSectionFound = false;
    TokenSequence tokenSequence = (TokenSequence) carrier.getData();
    for (Token token : tokenSequence) {
        String tokenText = token.getText().split(this.csvSeparator)[0];

        if (tokenText.contains("Literaturverzeichnis") || tokenText.contains("Quellennachweise")
                || tokenText.contains("References") || tokenText.contains("REFERENCES")
                || tokenText.contains("Notes") || tokenText.contains("Literatur")
                || tokenText.contains("LITERATURVERZEICHNIS")) {
            this.referenceSectionFound = true;
        }
        if (this.referenceSectionFound) {
            token.setFeatureValue(this.feature, 1.0);
        }
    }
    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
    TokenSequence tokenSequence = (TokenSequence) carrier.getData();
    int prevCount = 0;
    for (int i = 0; i < tokenSequence.size(); i++) {
        Token token = tokenSequence.get(i);
        String tokenText = token.getText().split(this.csvSeparator)[0];
        int count = 0;
        Matcher matcher = this.pattern.matcher(tokenText);
        while (matcher.find()) {
            count++;
        }
        // int count = StringUtils.countMatches(tokenText, this.subString);
        if (count < prevCount) {
            token.setFeatureValue(this.feature, 1.0);
        }
        prevCount = count;
    }
    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
@Override
public Instance pipe(Instance carrier) {

    TokenSequence targets = (TokenSequence) carrier.getTarget();

    for (int i = 0; i < targets.size(); i++) {

        Token target = targets.get(i);
        // System.out.println(t.getText());
        String targetLabel = target.getText();
        // System.out.println(lineWithoutFirst);
        // targetTokenSeq.add(lineSplit[0]);
        if (this.replacementMap.containsKey(targetLabel)) {
            target.setText(this.replacementMap.get(targetLabel));
        }
    }
    carrier.setTarget(targets);

    return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	if (ts.size() > 3
			&& (ts.get(2).getText().equals("-") || ts.get(3).getText().equals("-"))
			&& ts.get(1).getText().matches("[A-Z]+")) {
		String header = ts.get(1).getText();
		if (header.equals("PRESS"))				// Don't bother with "PRESS DIGEST" headers
			return carrier;
		String featureName = "HEADER="+header;
		for (int i = 0; i < ts.size(); i++) {
			Token t = ts.get(i);
			// Only apply this feature to capitalized words, because if we apply it to everything
			// we easily get an immense number of possible feature conjunctions, (e.g. every word
			// with each of these HEADER= features.
			if (t.getText().matches("^[A-Z].*"))
				t.setFeatureValue (featureName, 1.0);
		}
	}
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public Instance pipe(Instance carrier) {
	TokenSequence ts = (TokenSequence) carrier.getData();
	TokenSequence newTs = new TokenSequence();
	FeatureSequence labelSeq = new FeatureSequence(getTargetAlphabet());
	boolean lastWasSpace = true;
	StringBuffer sb = new StringBuffer();
	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		if (t.getText().equals(" "))
			lastWasSpace = true;
		else {
			sb.append(t.getText());
			newTs.add(t);
			labelSeq.add(lastWasSpace ? "start" : "notstart");
			lastWasSpace = false;
		}
	}
	if (isTargetProcessing())
		carrier.setTarget(labelSeq);
	carrier.setData(newTs);
	carrier.setSource(sb.toString());
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
	TokenSequence ts = (TokenSequence) carrier.getData();
	int tsSize = ts.size();
	for (int i = tsSize-1; i >= 0; i--) {
		Token t = ts.get (i);
		String text = t.getText();
		if (featureRegex != null && !featureRegex.matcher(text).matches())
			continue;
		for (int j = 0; j < i; j++) {
			if (ts.get(j).getText().equals(text)) {
				PropertyList.Iterator iter = ts.get(j).getFeatures().iterator();
				while (iter.hasNext()) {
					iter.next();
					String key = iter.getKey();
					if (filterRegex == null || (filterRegex.matcher(key).matches() ^ !includeFiltered))
						t.setFeatureValue (namePrefix+key, iter.getNumericValue());
				}
				break;
			}
			if (firstMentionName != null)
				t.setFeatureValue (firstMentionName, 1.0);
		}
	}
	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public static void testMultiTagSerialization () throws IOException, ClassNotFoundException
{
  Pipe origPipe = new SerialPipes (new Pipe[] {
          new SimpleTaggerSentence2TokenSequence (),
          new TokenText (),
          new RegexMatches ("digits", Pattern.compile ("[0-9]+")),
          new RegexMatches ("ampm", Pattern.compile ("[aApP][mM]")),
          new OffsetFeatureConjunction ("time",
                  new String[] { "digits", "ampm" },
                  new int[] { 0, 1 },
                  true),
          new PrintInputAndTarget (),
  });

  Pipe mtPipe = (Pipe) TestSerializable.cloneViaSerialization (origPipe);
  InstanceList mtLst = new InstanceList (mtPipe);
  mtLst.addThruPipe (new ArrayIterator (doc1));
  Instance mtInst = mtLst.get (0);
  TokenSequence mtTs = (TokenSequence) mtInst.getData ();
  assertEquals (6, mtTs.size ());
  assertEquals (1.0, mtTs.get (3).getFeatureValue ("time"), 1e-15);
  assertEquals (1.0, mtTs.get (4).getFeatureValue ("time"), 1e-15);
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public static void main (String[] args)
	{
		try {
			Pipe p = new SerialPipes (new Pipe[] {
				new Input2CharSequence (),
				new SGML2TokenSequence()
//				new SGML2TokenSequence (new CharSequenceLexer (Pattern.compile (".")), "O")
				});

			for (int i = 0; i < args.length; i++) {
				Instance carrier = p.instanceFrom(new Instance (new File(args[i]), null, null, null));
				TokenSequence data = (TokenSequence) carrier.getData();
				TokenSequence target = (TokenSequence) carrier.getTarget();
				logger.finer ("===");
				logger.info (args[i]);
				for (int j = 0; j < data.size(); j++)
					logger.info (target.get(j).getText()+" "+data.get(j).getText());
			}
		} catch (Exception e) {
			System.out.println (e);
			e.printStackTrace();
		}
	}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public Instance pipe (Instance carrier)
{
	String newTerm = null;
	TokenSequence tmpTS = new TokenSequence();
	TokenSequence ts = (TokenSequence) carrier.getData();

	for (int i = 0; i < ts.size(); i++) {
		Token t = ts.get(i);
		for(int j = 0; j < gramSizes.length; j++) {
			int len = gramSizes[j];
			if (len <= 0 || len > (i+1)) continue;
			if (len == 1) { tmpTS.add(t); continue; }
			newTerm = new String(t.getText());
			for(int k = 1; k < len; k++)
				newTerm = ts.get(i-k).getText() + "_" + newTerm;
			tmpTS.add(newTerm);
		}
	}

	carrier.setData(tmpTS);

	return carrier;
}
 

import cc.mallet.types.TokenSequence; //导入依赖的package包/类
public Instance pipe (Instance carrier) {
	TokenSequence ts = (TokenSequence) carrier.getData ();
	for (int i=0; i < ts.size(); i++) {
		Token t = ts.get (i);
		String[] values = t.getText().split("\\s+");
		for (int j=0; j < values.length; j++) {
			if (specifyFeatureNames) {
				String[] nameAndValue = values[j].split(nameValueSeparator);						
				if (nameAndValue.length != 2) { // no feature name. use token as feature.
					t.setFeatureValue ("Token="+values[j], 1.0);
				}
				else {
					t.setFeatureValue (nameAndValue[0], Double.parseDouble (nameAndValue[1]));						
				}
			}
			else if (realValued) {
				t.setFeatureValue ("Feature#" + j, Double.parseDouble (values[j]));
			}
			else
				t.setFeatureValue (values[j], 1.0);					
		}
	}
	carrier.setData (ts);
	return carrier;
}