Java Token.getSurface方法代码示例

import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
/** Attempts to translate a text using the word list. */
public String translate(Tokenizer tok, String text)
{
	StringBuilder res = new StringBuilder();
	
	for(Token t : tok.tokenize(text)) {
		if(Transliterator.hasAsianChar(t.getSurface())) {
			// attempt (partial) translation
			String s = t.getSurface();
			String r = StringUtils.replaceEach(s, srcEverywhere, trlEverywhere);
			r = replaceAnyOfAtStart(r, srcPrefix, trlPrefix);
			r = replaceAnyOfAtEnd(r, srcSuffix, trlSuffix);
			if(r != s && verbose > 1)
				System.out.println("translation list: translated " + s + " to " + r);
			res.append(r);				
		}
		else // tokens without asian chars can be ignored here and will be copied directly
			res.append(t.getSurface());
	}
	return res.toString();
}
 

import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
public static List<String> analyze(final String sentence) {
    Tokenizer search = getTokenizer();
    final List<Token> tokens = search.tokenize(sentence);
    final List<String> analiezed = Lists.newArrayList();
    for (final Token token : tokens) {
        final String word = token.getSurface();
        analiezed.add(word);
    }
    return analiezed;
}
 

import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
public String transliterate(String jaName) throws IOException {
   	String trName = jaName; 
   	
   	trName = prenormalize(trName);
   	
	// Use Kuromoji for preprocessing of kanji and for tokenization
	StringBuilder sb = new StringBuilder();
	for(Token t : tokKanji.tokenize(trName)) {
		if(hasKanji(t.getSurface()) && !(t.getReading().equals("*"))) {
			if(sb.length() > 0)
				sb.append(' ');
			sb.append(t.getReading());
		}
		/*if(hasKanji(t.getSurface()) && !(t.getLemmaReadingForm().equals("*"))) {
			if(sb.length() > 0)
				sb.append(' ');
			sb.append(t.getLemmaReadingForm());
		}*/
		else {
			String surf = t.getSurface();
			if(sb.length() > 0 && surf.length() > 0 && !Character.isWhitespace(surf.charAt(0)))
				sb.append(' ');
			sb.append(surf);
		}
		/*sb.append(t.getReading());
		sb.append('[');
		sb.append(t.getSurface());
		sb.append(']');
		sb.append(' ');*/
	}
	trName = sb.toString();
	
	// Use Kakasi for rest of conversion
	trName = trHiragana.doString(trName);
	trName = trKatakana.doString(trName);
	
	// Add spaces between words (but not within abbreviations)
	trName = trName.replaceAll("([a-z0-9])([A-Z])", "$1 $2");
	trName = postnormalize(trName);
	
	/*// debugging option: marking remaining asian chars in the output.  
	StringBuilder sbOut = new StringBuilder();
	for(char c : trName.toCharArray()) {
		sbOut.append(c);
		if(c > 0x200) {
			sbOut.append(String.format("[\\u%04x]", (int)c));
		}
			
	} 

	return sbOut.toString(); */
	return trName; 
}
 

import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
/**
 * @param builder
 */
private void setValue(PageBuilder builder)
{
    if (task.getKeepInput()) {
        for (Column inputColumn : inputSchema.getColumns()) {
            if (reader.isNull(inputColumn)) {
                builder.setNull(inputColumn);
                continue;
            }
            if (Types.STRING.equals(inputColumn.getType())) {
                builder.setString(inputColumn, reader.getString(inputColumn));
            }
            else if (Types.BOOLEAN.equals(inputColumn.getType())) {
                builder.setBoolean(inputColumn, reader.getBoolean(inputColumn));
            }
            else if (Types.DOUBLE.equals(inputColumn.getType())) {
                builder.setDouble(inputColumn, reader.getDouble(inputColumn));
            }
            else if (Types.LONG.equals(inputColumn.getType())) {
                builder.setLong(inputColumn, reader.getLong(inputColumn));
            }
            else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
                builder.setTimestamp(inputColumn, reader.getTimestamp(inputColumn));
            }
            else if (Types.JSON.equals(inputColumn.getType())) {
                builder.setJson(inputColumn, reader.getJson(inputColumn));
            }
        }
    }

    for (Column column : keyNameColumns) {
        final String source = reader.getString(column);
        List<Token> tokens = tokenizer.tokenize(source);
        logger.debug("{} => {}", source, tokens);
        for (Map<String, String> setting : task.getSettings()) {
            String suffix = setting.get("suffix");
            String method = setting.get("method");
            Column outputColumn = outputSchema.lookupColumn(column.getName() + MoreObjects.firstNonNull(suffix, ""));
            List<Value> outputs = Lists.newArrayList();
            for (Token token : tokens) {
                logger.debug("token => {}, {}", token, token.getAllFeatures());
                if (!isOkPartsOfSpeech(token)) {
                    continue;
                }
                String word = null;
                if ("base_form".equals(method)) {
                    word = MoreObjects.firstNonNull(token.getBaseForm(), token.getSurface());
                }
                else if ("reading".equals(method)) {
                    word = MoreObjects.firstNonNull(token.getReading(), token.getSurface());
                }
                else if ("surface_form".equals(method)) {
                    word = token.getSurface();
                }
                outputs.add(ValueFactory.newString(word));
            }
            if (outputColumn.getType().equals(Types.STRING)) {
                Joiner joiner = Joiner.on(MoreObjects.firstNonNull(setting.get("delimiter"), ",")).skipNulls();
                builder.setString(outputColumn, joiner.join(outputs));
            }
            else if (outputColumn.getType().equals(Types.JSON)) {
                builder.setJson(outputColumn, ValueFactory.newArray(outputs));
            }
        }
    }
}
 

import com.atilika.kuromoji.ipadic.Token; //导入方法依赖的package包/类
private String getToken(int i) {
    Token t = tokens.get(i);
    String ret = (useBaseForm) ? t.getBaseForm() : t.getSurface();
    return (preProcessor == null) ? ret : preProcessor.preProcess(ret);
}