本文整理汇总了Java中com.atilika.kuromoji.ipadic.Tokenizer类的典型用法代码示例。如果您正苦于以下问题:Java Tokenizer类的具体用法?Java Tokenizer怎么用?Java Tokenizer使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Tokenizer类属于com.atilika.kuromoji.ipadic包,在下文中一共展示了Tokenizer类的8个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: translate
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/** Attempts to translate a text using the word list. */
public String translate(Tokenizer tok, String text)
{
StringBuilder res = new StringBuilder();
for(Token t : tok.tokenize(text)) {
if(Transliterator.hasAsianChar(t.getSurface())) {
// attempt (partial) translation
String s = t.getSurface();
String r = StringUtils.replaceEach(s, srcEverywhere, trlEverywhere);
r = replaceAnyOfAtStart(r, srcPrefix, trlPrefix);
r = replaceAnyOfAtEnd(r, srcSuffix, trlSuffix);
if(r != s && verbose > 1)
System.out.println("translation list: translated " + s + " to " + r);
res.append(r);
}
else // tokens without asian chars can be ignored here and will be copied directly
res.append(t.getSurface());
}
return res.toString();
}
示例2: Transliterator
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
public Transliterator()
{
tokKanji = new Tokenizer();
trHiragana = new Kakasi();
trHiragana.setupHiraganaConverter(Kakasi.ASCII);
trHiragana.setRomajiCapitalizeMode(true);
trKatakana = new Kakasi();
trKatakana.setupKatakanaConverter(Kakasi.ASCII);
trKatakana.setRomajiCapitalizeMode(true);
}
示例3: getTokenizer
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
private static synchronized Tokenizer getTokenizer() {
if (token == null) {
final Tokenizer.Builder builder = new Tokenizer.Builder();
builder.mode(Tokenizer.Mode.SEARCH);
token = builder.build();
}
return token;
}
示例4: analyze
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
public static List<String> analyze(final String sentence) {
Tokenizer search = getTokenizer();
final List<Token> tokens = search.tokenize(sentence);
final List<String> analiezed = Lists.newArrayList();
for (final Token token : tokens) {
final String word = token.getSurface();
analiezed.add(word);
}
return analiezed;
}
示例5: getTokenizer
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
Tokenizer getTokenizer() {
return tokKanji;
}
示例6: Tokenizer
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/**
* Construct a default tokenizer
*/
public Tokenizer() {
this(new Builder());
}
示例7: JapaneseTokenizer
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/**
* Tokenize the string with Kuromoji, optionally using baseForms.
*
* Note: It is safe to create new instances from multiple threads.
* @param kuromoji The kuromoji instance.
* @param toTokenize The string to tokenize.
* @param useBaseForm normalize conjugations "走った" -> "走る" instead of "走っ"
*/
public JapaneseTokenizer(Tokenizer kuromoji, String toTokenize, boolean useBaseForm) {
this.useBaseForm = useBaseForm;
this.tokens = kuromoji.tokenize(toTokenize);
this.tokenCount = this.tokens.size();
this.currentToken = 0;
}
示例8: build
import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/**
* Creates the custom tokenizer instance
*
* @return tokenizer instance, not null
*/
@Override
public Tokenizer build() {
return new Tokenizer(this);
}