本文整理汇总了Java中cc.mallet.types.TokenSequence.size方法的典型用法代码示例。如果您正苦于以下问题：Java TokenSequence.size方法的具体用法？Java TokenSequence.size怎么用？Java TokenSequence.size使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类cc.mallet.types.TokenSequence的用法示例。
在下文中一共展示了TokenSequence.size方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData();
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
int splitLength = t.getText().split("\t").length;
if (splitLength == this.minLineLength) {
t.setText("O\t" + t.getText());
} else {
if (splitLength != (this.minLineLength + 1)) {
System.err.println("input line does not have length " + this.minLineLength + " or "
+ (this.minLineLength + 1) + " but " + splitLength + ": " + t.getText());
}
}
}
carrier.setData(ts);
return carrier;
}
示例2: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
TokenSequence tokenSequence = (TokenSequence) carrier.getData();
for (int i = 0; i < tokenSequence.size(); i++) {
Token token = tokenSequence.get(i);
String tokenText = token.getText().split(this.csvSeparator)[0];
int count = 0;
Matcher matcher = this.pattern.matcher(tokenText);
while (matcher.find()) {
count++;
}
// int count = StringUtils.countMatches(tokenText, this.subString);
if (count > 0) {
// token.setFeatureValue(this.feature + "=" + count, 1.0);
token.setFeatureValue(this.feature, count);
}
}
return carrier;
}
示例3: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData();
TokenSequence targetTokenSeq = new TokenSequence(ts.size());
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
// System.out.println(t.getText());
String lineWithoutFirst = t.getText().replaceFirst("[^\\t]*\t", "");
// System.out.println(lineWithoutFirst);
// targetTokenSeq.add(lineSplit[0]);
targetTokenSeq.add(t.getText().split("\t")[0]);
t.setText(lineWithoutFirst);
}
carrier.setTarget(targetTokenSeq);
carrier.setData(ts);
return carrier;
}
示例4: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
TokenSequence tokenSequence = (TokenSequence) carrier.getData();
int prevCount = 0;
for (int i = 0; i < tokenSequence.size(); i++) {
Token token = tokenSequence.get(i);
String tokenText = token.getText().split(this.csvSeparator)[0];
int count = 0;
Matcher matcher = this.pattern.matcher(tokenText);
while (matcher.find()) {
count++;
}
// int count = StringUtils.countMatches(tokenText, this.subString);
if (count < prevCount) {
token.setFeatureValue(this.feature, 1.0);
}
prevCount = count;
}
return carrier;
}
示例5: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
@Override
public Instance pipe(Instance carrier) {
TokenSequence targets = (TokenSequence) carrier.getTarget();
for (int i = 0; i < targets.size(); i++) {
Token target = targets.get(i);
// System.out.println(t.getText());
String targetLabel = target.getText();
// System.out.println(lineWithoutFirst);
// targetTokenSeq.add(lineSplit[0]);
if (this.replacementMap.containsKey(targetLabel)) {
target.setText(this.replacementMap.get(targetLabel));
}
}
carrier.setTarget(targets);
return carrier;
}
示例6: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe(Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData();
TokenSequence newTs = new TokenSequence();
FeatureSequence labelSeq = new FeatureSequence(getTargetAlphabet());
boolean lastWasSpace = true;
StringBuffer sb = new StringBuffer();
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
if (t.getText().equals(" "))
lastWasSpace = true;
else {
sb.append(t.getText());
newTs.add(t);
labelSeq.add(lastWasSpace ? "start" : "notstart");
lastWasSpace = false;
}
}
if (isTargetProcessing())
carrier.setTarget(labelSeq);
carrier.setData(newTs);
carrier.setSource(sb.toString());
return carrier;
}
示例7: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
TokenSequence ts = (TokenSequence) carrier.getData();
int tsSize = ts.size();
for (int i = tsSize-1; i >= 0; i--) {
Token t = ts.get (i);
String text = t.getText();
if (featureRegex != null && !featureRegex.matcher(text).matches())
continue;
for (int j = 0; j < i; j++) {
if (ts.get(j).getText().equals(text)) {
PropertyList.Iterator iter = ts.get(j).getFeatures().iterator();
while (iter.hasNext()) {
iter.next();
String key = iter.getKey();
if (filterRegex == null || (filterRegex.matcher(key).matches() ^ !includeFiltered))
t.setFeatureValue (namePrefix+key, iter.getNumericValue());
}
break;
}
if (firstMentionName != null)
t.setFeatureValue (firstMentionName, 1.0);
}
}
return carrier;
}
示例8: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
TokenSequence ts = (TokenSequence) carrier.getData();
// xxx This doesn't seem so efficient. Perhaps have TokenSequence
// use a LinkedList, and remove Tokens from it? -?
// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
TokenSequence ret = new TokenSequence ();
Token prevToken = null;
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
if (! stoplist.contains (caseSensitive ? t.getText() : t.getText().toLowerCase())) {
// xxx Should we instead make and add a copy of the Token?
ret.add (t);
prevToken = t;
} else if (markDeletions && prevToken != null)
prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
}
carrier.setData(ret);
return carrier;
}
示例9: main
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public static void main (String[] args)
{
try {
Pipe p = new SerialPipes (new Pipe[] {
new Input2CharSequence (),
new SGML2TokenSequence()
// new SGML2TokenSequence (new CharSequenceLexer (Pattern.compile (".")), "O")
});
for (int i = 0; i < args.length; i++) {
Instance carrier = p.instanceFrom(new Instance (new File(args[i]), null, null, null));
TokenSequence data = (TokenSequence) carrier.getData();
TokenSequence target = (TokenSequence) carrier.getTarget();
logger.finer ("===");
logger.info (args[i]);
for (int j = 0; j < data.size(); j++)
logger.info (target.get(j).getText()+" "+data.get(j).getText());
}
} catch (Exception e) {
System.out.println (e);
e.printStackTrace();
}
}
示例10: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
String newTerm = null;
TokenSequence tmpTS = new TokenSequence();
TokenSequence ts = (TokenSequence) carrier.getData();
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
for(int j = 0; j < gramSizes.length; j++) {
int len = gramSizes[j];
if (len <= 0 || len > (i+1)) continue;
if (len == 1) { tmpTS.add(t); continue; }
newTerm = new String(t.getText());
for(int k = 1; k < len; k++)
newTerm = ts.get(i-k).getText() + "_" + newTerm;
tmpTS.add(newTerm);
}
}
carrier.setData(tmpTS);
return carrier;
}
示例11: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData ();
for (int i=0; i < ts.size(); i++) {
Token t = ts.get (i);
String[] values = t.getText().split("\\s+");
for (int j=0; j < values.length; j++) {
if (specifyFeatureNames) {
String[] nameAndValue = values[j].split(nameValueSeparator);
if (nameAndValue.length != 2) { // no feature name. use token as feature.
t.setFeatureValue ("Token="+values[j], 1.0);
}
else {
t.setFeatureValue (nameAndValue[0], Double.parseDouble (nameAndValue[1]));
}
}
else if (realValued) {
t.setFeatureValue ("Feature#" + j, Double.parseDouble (values[j]));
}
else
t.setFeatureValue (values[j], 1.0);
}
}
carrier.setData (ts);
return carrier;
}
开发者ID:kostagiolasn，项目名称:NucleosomePatternClassifier，代码行数:26，代码来源:TokenSequenceParseFeatureString.java
示例12: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
TokenSequence ts = (TokenSequence) carrier.getData();
// xxx This doesn't seem so efficient. Perhaps have TokenSequence
// use a LinkedList, and remove Tokens from it? -?
// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
TokenSequence ret = new TokenSequence ();
Token prevToken = null;
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
String s = t.getText();
if (CharSequenceLexer.LEX_ALPHA.matcher(s).matches()) {
ret.add (t);
prevToken = t;
} else if (markDeletions && prevToken != null)
prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
}
carrier.setData(ret);
return carrier;
}
示例13: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
TokenSequence ts = (TokenSequence) carrier.getData();
TokenSequence targetTokenSeq = new TokenSequence (ts.size());
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
Matcher matcher = regex.matcher (t.getText());
if (matcher.matches()) {
targetTokenSeq.add (matcher.group(targetGroup));
t.setText (matcher.group (dataGroup));
} else {
logger.warning ("Skipping token: No match of "+regex.pattern()
+" at token #"+i+" with text "+t.getText());
}
}
carrier.setTarget(targetTokenSeq);
carrier.setData(ts);
return carrier;
}
开发者ID:kostagiolasn，项目名称:NucleosomePatternClassifier，代码行数:20，代码来源:TokenSequenceMatchDataAndTarget.java
示例14: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
TokenSequence ts = (TokenSequence) carrier.getData();
BigFeatureSequence ret =
new BigFeatureSequence ((BigAlphabet)getDataAlphabet(), ts.size());
long millis = System.currentTimeMillis();
for (int i = 0; i < ts.size(); i++) {
ret.add (ts.get(i).getText());
}
//System.out.println(this.getClass().getCanonicalName() + "x===x" + (System.currentTimeMillis()-millis));
carrier.setData(ret);
//carrier.lock();
return carrier;
}
示例15: pipe
import cc.mallet.types.TokenSequence; //导入方法依赖的package包/类
public Instance pipe (Instance carrier)
{
long millis = System.currentTimeMillis();
TokenSequence ts = (TokenSequence) carrier.getData();
// xxx This doesn't seem so efficient. Perhaps have TokenSequence
// use a LinkedList, and remove Tokens from it? -?
// But a LinkedList implementation of TokenSequence would be quite inefficient -AKM
TokenSequence ret = new TokenSequence ();
Token prevToken = null;
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
if (! stoplist.contains (caseSensitive ? t.getText() : t.getText().toLowerCase())) {
// xxx Should we instead make and add a copy of the Token?
ret.add (t);
prevToken = t;
} else if (markDeletions && prevToken != null)
prevToken.setProperty (FeatureSequenceWithBigrams.deletionMark, t.getText());
}
carrier.setData(ret);
//System.out.println(this.getClass().getCanonicalName() + "===" + (System.currentTimeMillis()-millis));
return carrier;
}