本文整理汇总了Java中com.twitter.penguin.korean.KoreanTokenJava类的典型用法代码示例。如果您正苦于以下问题：Java KoreanTokenJava类的具体用法？Java KoreanTokenJava怎么用？Java KoreanTokenJava使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
KoreanTokenJava类属于com.twitter.penguin.korean包，在下文中一共展示了KoreanTokenJava类的3个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: setAttributes
import com.twitter.penguin.korean.KoreanTokenJava; //导入依赖的package包/类
/**
* Set token attributes such as term, start offset, end offset, type.
* @param token
*/
protected void setAttributes(KoreanTokenJava token){
char[] buf = token.getText().toCharArray();
termAtt.copyBuffer(buf, 0, buf.length);
offset = token.getOffset();
offsetAtt.setOffset(correctOffset(offset), finalOffset = (correctOffset(offset) + token.getLength()));
typeAtt.setType(token.getPos().toString());
}
示例2: KoreanTokenizer
import com.twitter.penguin.korean.KoreanTokenJava; //导入依赖的package包/类
public KoreanTokenizer(String toTokenize) {
// need normalize?
// Tokenize
Seq<com.twitter.penguin.korean.tokenizer.KoreanTokenizer.KoreanToken> tokens =
TwitterKoreanProcessorJava.tokenize(toTokenize);
tokenList = new ArrayList<String>();
Iterator<KoreanTokenJava> iter = TwitterKoreanProcessorJava.tokensToJavaKoreanTokenList(tokens).iterator();
while (iter.hasNext()) {
tokenList.add(iter.next().getText());
}
tokenIter = tokenList.iterator();
}
示例3: main
import com.twitter.penguin.korean.KoreanTokenJava; //导入依赖的package包/类
public static void main(String[] args) {
String text = "대숲 어쩌죠...? 짝사랑을 시작하게 된 거 같아요ㅜㅜㅜㅠㅠㅠㅠㅠ";
// Normalize
CharSequence normalized = TwitterKoreanProcessorJava.normalize(text);
System.out.println(normalized);
// 한국어를 처리하는 예시입니다ㅋㅋ #한국어
// Tokenize
Seq<KoreanTokenizer.KoreanToken> tokens = TwitterKoreanProcessorJava.tokenize(normalized);
//System.out.println(TwitterKoreanProcessorJava.tokensToJavaStringList(tokens));
// [한국어, 를, 처리, 하는, 예시, 입니, 다, ㅋㅋ, #한국어]
// System.out.println(TwitterKoreanProcessorJava.tokensToJavaKoreanTokenList(tokens));
// [한국어(Noun: 0, 3), 를(Josa: 3, 1), (Space: 4, 1), 처리(Noun: 5, 2), 하는(Verb: 7, 2), (Space: 9, 1), 예시(Noun: 10, 2), 입니(Adjective: 12, 2), 다(Eomi: 14, 1), ㅋㅋ(KoreanParticle: 15, 2), (Space: 17, 1), #한국어(Hashtag: 18, 4)]
// Stemming
Seq<KoreanTokenizer.KoreanToken> stemmed = TwitterKoreanProcessorJava.stem(tokens);
System.out.println(TwitterKoreanProcessorJava.tokensToJavaStringList(stemmed));
// [한국어, 를, 처리, 하다, 예시, 이다, ㅋㅋ, #한국어]
List<KoreanTokenJava> change = TwitterKoreanProcessorJava.tokensToJavaKoreanTokenList(stemmed);
LinkedList<String> change_list = new LinkedList<String>();
for(int i=0; i<change.size(); i++) {
KoreanTokenJava str = change.get(i);
String str2 = str.toString();
str2 = str2.replaceAll(":", "");
str2 = str2.replaceAll("[0-9]", "");
str2 = str2.replaceAll(" , ", "");
change_list.add(str2);
}
System.out.println(change_list);
// [한국어(Noun: 0, 3), 를(Josa: 3, 1), (Space: 4, 1), 처리(Noun: 5, 2), 하다(Verb: 7, 2), (Space: 9, 1), 예시(Noun: 10, 2), 이다(Adjective: 12, 3), ㅋㅋ(KoreanParticle: 15, 2), (Space: 17, 1), #한국어(Hashtag: 18, 4)]
/*
// Phrase extraction
List<KoreanPhraseExtractor.KoreanPhrase> phrases = TwitterKoreanProcessorJava.extractPhrases(tokens, true, true);
System.out.println(phrases);
// [한국어(Noun: 0, 3), 처리(Noun: 5, 2), 처리하는 예시(Noun: 5, 7), 예시(Noun: 10, 2), #한국어(Hashtag: 18, 4)]
*/
}