本文整理汇总了Java中com.atilika.kuromoji.ipadic.Token.getSurface方法的典型用法代码示例。如果您正苦于以下问题:Java Token.getSurface方法的具体用法?Java Token.getSurface怎么用?Java Token.getSurface使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类com.atilika.kuromoji.ipadic.Token
的用法示例。
在下文中一共展示了Token.getSurface方法的5个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: translate
import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
/** Attempts to translate a text using the word list. */
public String translate(Tokenizer tok, String text)
{
StringBuilder res = new StringBuilder();
for(Token t : tok.tokenize(text)) {
if(Transliterator.hasAsianChar(t.getSurface())) {
// attempt (partial) translation
String s = t.getSurface();
String r = StringUtils.replaceEach(s, srcEverywhere, trlEverywhere);
r = replaceAnyOfAtStart(r, srcPrefix, trlPrefix);
r = replaceAnyOfAtEnd(r, srcSuffix, trlSuffix);
if(r != s && verbose > 1)
System.out.println("translation list: translated " + s + " to " + r);
res.append(r);
}
else // tokens without asian chars can be ignored here and will be copied directly
res.append(t.getSurface());
}
return res.toString();
}
示例2: analyze
import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
public static List<String> analyze(final String sentence) {
Tokenizer search = getTokenizer();
final List<Token> tokens = search.tokenize(sentence);
final List<String> analiezed = Lists.newArrayList();
for (final Token token : tokens) {
final String word = token.getSurface();
analiezed.add(word);
}
return analiezed;
}
示例3: transliterate
import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
public String transliterate(String jaName) throws IOException {
String trName = jaName;
trName = prenormalize(trName);
// Use Kuromoji for preprocessing of kanji and for tokenization
StringBuilder sb = new StringBuilder();
for(Token t : tokKanji.tokenize(trName)) {
if(hasKanji(t.getSurface()) && !(t.getReading().equals("*"))) {
if(sb.length() > 0)
sb.append(' ');
sb.append(t.getReading());
}
/*if(hasKanji(t.getSurface()) && !(t.getLemmaReadingForm().equals("*"))) {
if(sb.length() > 0)
sb.append(' ');
sb.append(t.getLemmaReadingForm());
}*/
else {
String surf = t.getSurface();
if(sb.length() > 0 && surf.length() > 0 && !Character.isWhitespace(surf.charAt(0)))
sb.append(' ');
sb.append(surf);
}
/*sb.append(t.getReading());
sb.append('[');
sb.append(t.getSurface());
sb.append(']');
sb.append(' ');*/
}
trName = sb.toString();
// Use Kakasi for rest of conversion
trName = trHiragana.doString(trName);
trName = trKatakana.doString(trName);
// Add spaces between words (but not within abbreviations)
trName = trName.replaceAll("([a-z0-9])([A-Z])", "$1 $2");
trName = postnormalize(trName);
/*// debugging option: marking remaining asian chars in the output.
StringBuilder sbOut = new StringBuilder();
for(char c : trName.toCharArray()) {
sbOut.append(c);
if(c > 0x200) {
sbOut.append(String.format("[\\u%04x]", (int)c));
}
}
return sbOut.toString(); */
return trName;
}
示例4: setValue
import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
/**
* @param builder
*/
private void setValue(PageBuilder builder)
{
if (task.getKeepInput()) {
for (Column inputColumn : inputSchema.getColumns()) {
if (reader.isNull(inputColumn)) {
builder.setNull(inputColumn);
continue;
}
if (Types.STRING.equals(inputColumn.getType())) {
builder.setString(inputColumn, reader.getString(inputColumn));
}
else if (Types.BOOLEAN.equals(inputColumn.getType())) {
builder.setBoolean(inputColumn, reader.getBoolean(inputColumn));
}
else if (Types.DOUBLE.equals(inputColumn.getType())) {
builder.setDouble(inputColumn, reader.getDouble(inputColumn));
}
else if (Types.LONG.equals(inputColumn.getType())) {
builder.setLong(inputColumn, reader.getLong(inputColumn));
}
else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
builder.setTimestamp(inputColumn, reader.getTimestamp(inputColumn));
}
else if (Types.JSON.equals(inputColumn.getType())) {
builder.setJson(inputColumn, reader.getJson(inputColumn));
}
}
}
for (Column column : keyNameColumns) {
final String source = reader.getString(column);
List<Token> tokens = tokenizer.tokenize(source);
logger.debug("{} => {}", source, tokens);
for (Map<String, String> setting : task.getSettings()) {
String suffix = setting.get("suffix");
String method = setting.get("method");
Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
List<Value> outputs = Lists.newArrayList();
for (Token token : tokens) {
logger.debug("token => {}, {}", token, token.getAllFeatures());
if (!isOkPartsOfSpeech(token)) {
continue;
}
String word = null;
if ("base_form".equals(method)) {
word = MoreObjects.firstNonNull(token.getBaseForm(), token.getSurface());
}
else if ("reading".equals(method)) {
word = MoreObjects.firstNonNull(token.getReading(), token.getSurface());
}
else if ("surface_form".equals(method)) {
word = token.getSurface();
}
outputs.add(ValueFactory.newString(word));
}
if (outputColumn.getType().equals(Types.STRING)) {
Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
builder.setString(outputColumn, joiner.join(outputs));
}
else if (outputColumn.getType().equals(Types.JSON)) {
builder.setJson(outputColumn, ValueFactory.newArray(outputs));
}
}
}
}
示例5: getToken
import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
private String getToken(int i) {
Token t = tokens.get(i);
String ret = (useBaseForm) ? t.getBaseForm() : t.getSurface();
return (preProcessor == null) ? ret : preProcessor.preProcess(ret);
}