本文整理汇总了Java中org.languagetool.Language.getSentenceTokenizer方法的典型用法代码示例。如果您正苦于以下问题：Java Language.getSentenceTokenizer方法的具体用法？Java Language.getSentenceTokenizer怎么用？Java Language.getSentenceTokenizer使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.languagetool.Language的用法示例。
在下文中一共展示了Language.getSentenceTokenizer方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: CommonCrawlToNgram
import org.languagetool.Language; //导入方法依赖的package包/类
CommonCrawlToNgram(Language language, File input, File indexTopDir, File evalFile) throws IOException {
this.input = input;
this.indexTopDir = indexTopDir;
this.evalFile = evalFile;
this.sentenceTokenizer = language.getSentenceTokenizer();
this.wordTokenizer = new GoogleStyleWordTokenizer();
indexes.put(1, new LuceneLiveIndex(new File(indexTopDir, "1grams")));
indexes.put(2, new LuceneLiveIndex(new File(indexTopDir, "2grams")));
indexes.put(3, new LuceneLiveIndex(new File(indexTopDir, "3grams")));
}
示例2: CommonCrawlToNgram3
import org.languagetool.Language; //导入方法依赖的package包/类
CommonCrawlToNgram3(Language language, File input, File outputDir) throws IOException {
this.input = input;
this.sentenceTokenizer = language.getSentenceTokenizer();
this.wordTokenizer = new GoogleStyleWordTokenizer();
ngramSizeToWriter.put(1, new FileWriter(new File(outputDir, "unigrams.csv")));
ngramSizeToWriter.put(2, new FileWriter(new File(outputDir, "bigrams.csv")));
ngramSizeToWriter.put(3, new FileWriter(new File(outputDir, "trigrams.csv")));
}
示例3: WikipediaSentenceSource
import org.languagetool.Language; //导入方法依赖的package包/类
/** @since 3.0 */
public WikipediaSentenceSource(InputStream xmlInput, Language language, Pattern filter) {
super(language, filter);
textFilter.enableMapping(false); // improves performance
try {
System.setProperty("jdk.xml.totalEntitySizeLimit", String.valueOf(Integer.MAX_VALUE)); // see https://github.com/dbpedia/extraction-framework/issues/487
XMLInputFactory factory = XMLInputFactory.newInstance();
reader = factory.createXMLEventReader(xmlInput);
sentenceTokenizer = language.getSentenceTokenizer();
sentences = new ArrayList<>();
this.language = language;
} catch (XMLStreamException e) {
throw new RuntimeException(e);
}
}
示例4: Indexer
import org.languagetool.Language; //导入方法依赖的package包/类
public Indexer(Directory dir, Language language, Analyzer analyzer) {
try {
IndexWriterConfig writerConfig = getIndexWriterConfig(analyzer);
writerConfig.setOpenMode(OpenMode.CREATE);
writer = new IndexWriter(dir, writerConfig);
sentenceTokenizer = language.getSentenceTokenizer();
} catch (Exception e) {
throw new RuntimeException(e);
}
}