Java Tokenizer类代码示例

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/** Attempts to translate a text using the word list. */
public String translate(Tokenizer tok, String text)
{
	StringBuilder res = new StringBuilder();
	
	for(Token t : tok.tokenize(text)) {
		if(Transliterator.hasAsianChar(t.getSurface())) {
			// attempt (partial) translation
			String s = t.getSurface();
			String r = StringUtils.replaceEach(s, srcEverywhere, trlEverywhere);
			r = replaceAnyOfAtStart(r, srcPrefix, trlPrefix);
			r = replaceAnyOfAtEnd(r, srcSuffix, trlSuffix);
			if(r != s && verbose > 1)
				System.out.println("translation list: translated " + s + " to " + r);
			res.append(r);				
		}
		else // tokens without asian chars can be ignored here and will be copied directly
			res.append(t.getSurface());
	}
	return res.toString();
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
public Transliterator()
{
   	tokKanji = new Tokenizer(); 
   	
   	trHiragana = new Kakasi();
   	trHiragana.setupHiraganaConverter(Kakasi.ASCII);
   	trHiragana.setRomajiCapitalizeMode(true);
   	
   	trKatakana = new Kakasi();
   	trKatakana.setupKatakanaConverter(Kakasi.ASCII);
   	trKatakana.setRomajiCapitalizeMode(true);		
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
private static synchronized Tokenizer getTokenizer() {
    if (token == null) {
        final Tokenizer.Builder builder = new Tokenizer.Builder();
        builder.mode(Tokenizer.Mode.SEARCH);
        token = builder.build();
    }
    return token;
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
public static List<String> analyze(final String sentence) {
    Tokenizer search = getTokenizer();
    final List<Token> tokens = search.tokenize(sentence);
    final List<String> analiezed = Lists.newArrayList();
    for (final Token token : tokens) {
        final String word = token.getSurface();
        analiezed.add(word);
    }
    return analiezed;
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
Tokenizer getTokenizer() {
	return tokKanji;
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/**
 * Construct a default tokenizer
 */
public Tokenizer() {
    this(new Builder());
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/**
 * Tokenize the string with Kuromoji, optionally using baseForms.
 *
 * Note: It is safe to create new instances from multiple threads.
 * @param kuromoji The kuromoji instance.
 * @param toTokenize The string to tokenize.
 * @param useBaseForm normalize conjugations "走った" -> "走る" instead of "走っ"
 */
public JapaneseTokenizer(Tokenizer kuromoji, String toTokenize, boolean useBaseForm) {
    this.useBaseForm = useBaseForm;
    this.tokens = kuromoji.tokenize(toTokenize);
    this.tokenCount = this.tokens.size();
    this.currentToken = 0;
}
 

import com.atilika.kuromoji.ipadic.Tokenizer; //导入依赖的package包/类
/**
 * Creates the custom tokenizer instance
 *
 * @return tokenizer instance, not null
 */
@Override
public Tokenizer build() {
    return new Tokenizer(this);
}