Java AnalysisService.tokenFilter方法代码示例

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testTokenizerFilter() throws IOException {

    String source = "eins zwei drei";

    String[] expected = {
            "d800000000001214",
            "4a00000000000000",
            "c0000000000a100"
    };
    AnalysisService analysisService = MapperTestUtils.analysisService("eudex_analysis.json");
    TokenFilterFactory tokenFilterFactory = analysisService.tokenFilter("my_phonetic");
    Tokenizer tokenizer = analysisService.tokenizer("my_phonetic").create();
    tokenizer.setReader(new StringReader(source));
    assertSimpleTSOutput(tokenFilterFactory.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testOpenNLPAnalysis() throws Exception {
    AnalysisService analysisService = MapperTestUtils.analysisService(getResource());
    TokenizerFactory tokenizerFactory = analysisService.tokenizer("opennlp");
    MatcherAssert.assertThat(tokenizerFactory, instanceOf(OpenNLPTokenizerFactory.class));
    this.tokenizerFactory = (OpenNLPTokenizerFactory) tokenizerFactory;
    TokenFilterFactory filterFactory = analysisService.tokenFilter("opennlp");
    MatcherAssert.assertThat(filterFactory, instanceOf(OpenNLPTokenFilterFactory.class));
    this.filterFactory = (OpenNLPTokenFilterFactory) filterFactory;

    TokenFilterFactory filterpayloadsfilterFactory = analysisService.tokenFilter("filterpayloads");
    MatcherAssert.assertThat(filterpayloadsfilterFactory, instanceOf(FilterPayloadsTokenFilterFactory.class));
    TokenFilterFactory strippayloadsfilterFactory = analysisService.tokenFilter("strippayloads");
    MatcherAssert.assertThat(strippayloadsfilterFactory, instanceOf(StripPayloadsTokenFilterFactory.class));

    analyze();
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testTwo() throws IOException {

    String source = "Das sind Autos, die Nudeln transportieren.";

    String[] expected = {
            "Das",
            "Das",
            "sind",
            "sind",
            "Autos",
            "Auto",
            "die",
            "der",
            "Nudeln",
            "Nudel",
            "transportieren",
            "transportieren"
    };
    AnalysisService analysisService = MapperTestUtils.analysisService();
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("baseform");
    Tokenizer tokenizer = analysisService.tokenizer("standard").create();
    tokenizer.setReader(new StringReader(source));
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testThree() throws IOException {

    String source = "wurde zum tollen gemacht";

    String[] expected = {
            "wurde",
            "werden",
            "zum",
            "zum",
            "tollen",
            "tollen",
            "gemacht",
            "machen"
    };
    AnalysisService analysisService = MapperTestUtils.analysisService();
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("baseform");
    Tokenizer tokenizer = analysisService.tokenizer("standard").create();
    tokenizer.setReader(new StringReader(source));
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void test() throws IOException {

    String source = "Ein schöner Tag in Köln im Café an der Straßenecke";

    String[] expected = {
        "Ein",
        "schoner",
        "Tag",
        "in",
        "Koln",
        "im",
        "Café",
        "an",
        "der",
        "Strassenecke"
    };

    AnalysisService analysisService = createAnalysisService();
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("umlaut");
    Tokenizer tokenizer = analysisService.tokenizer("standard").create(new StringReader(source));
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testTwo() throws IOException {
    AnalysisService analysisService = createAnalysisService();
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("baseform");

    String source = "Das sind Autos, die Nudeln transportieren.";

    String[] expected = {
            "Das",
            "Das",
            "sind",
            "sind",
            "Autos",
            "Auto",
            "die",
            "der",
            "Nudeln",
            "Nudel",
            "transportieren",
            "transportieren"
    };

    Tokenizer tokenizer = new StandardTokenizer(Version.LATEST, new StringReader(source));
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testOne() throws IOException {

    String source = "Das ist ein Bindestrich-Wort.";

    String[] expected = {
            "Das",
            "ist",
            "ein",
            "Bindestrich-Wort",
            "BindestrichWort",
            "Wort",
            "Bindestrich"
    };
    AnalysisService analysisService = createAnalysisService();
    Tokenizer tokenizer = analysisService.tokenizer("my_hyphen_tokenizer").create(new StringReader(source));
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("hyphen");
    TokenStream tokenStream = tokenFilter.create(tokenizer);
    assertSimpleTSOutput(tokenStream, expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testTwo() throws IOException {

    String source = "Das E-Book muss dringend zum Buchbinder.";

    String[] expected = {
            "Das",
            "E-Book",
            "EBook",
            "Book",
            "muss",
            "dringend",
            "zum",
            "Buchbinder"
    };
    AnalysisService analysisService = createAnalysisService();
    Tokenizer tokenizer = analysisService.tokenizer("my_icu_tokenizer").create(new StringReader(source));
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testFour() throws IOException {

    String source = "So wird's was: das Elasticsearch-Buch erscheint beim O'Reilly-Verlag.";

    String[] expected = {
            "So",
            "wird's",
            "was",
            "das",
            "Elasticsearch-Buch",
            "ElasticsearchBuch",
            "Buch",
            "Elasticsearch",
            "erscheint",
            "beim",
            "O'Reilly-Verlag"
    };
    AnalysisService analysisService = createAnalysisService();
    Tokenizer tokenizer = analysisService.tokenizer("my_hyphen_tokenizer").create(new StringReader(source));
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testSix() throws IOException {

    String source = "E-Book";

    String[] expected = {
            "E-Book",
            "EBook",
            "Book"
    };

    AnalysisService analysisService = createAnalysisService();
    Tokenizer tokenizer = analysisService.tokenizer("my_hyphen_tokenizer").create(new StringReader(source));
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testSeven() throws IOException {
    String source = "Procter & Gamble ist nicht schwarz - weiss";

    String[] expected = {
            "Procter",
            "Gamble",
            "ist",
            "nicht",
            "schwarz",
            "weiss"
    };

    AnalysisService analysisService = createAnalysisService();
    Tokenizer tokenizer = analysisService.tokenizer("my_hyphen_tokenizer").create(new StringReader(source));
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testDefaultsRSLPAnalysis() throws IOException {
    AnalysisService analysisService = createAnalysisService(new Index("test", "_na_"), Settings.EMPTY, new AnalysisMetaphonePlugin());

    TokenFilterFactory tokenizerFactory = analysisService.tokenFilter("br_metaphone");

    MatcherAssert.assertThat(tokenizerFactory, instanceOf(MetaphoneTokenFilterFactory.class));
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testMetaphoneWords() throws Exception {
    Index index = new Index("test", "_na_");
    Settings settings = Settings.builder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put("index.analysis.filter.myStemmer.type", "br_metaphone")
            .build();

    AnalysisService analysisService = createAnalysisService(index, settings, new AnalysisMetaphonePlugin());

    TokenFilterFactory filterFactory = analysisService.tokenFilter("br_metaphone");

    Tokenizer tokenizer = new KeywordTokenizer();
    
    Map<String,String> words = buildWordList();
    
    Set<String> inputWords = words.keySet();
    for(String word : inputWords) {
        tokenizer.setReader(new StringReader(word));
        TokenStream ts = filterFactory.create(tokenizer);

        CharTermAttribute term1 = ts.addAttribute(CharTermAttribute.class);
        ts.reset();
        assertThat(ts.incrementToken(), equalTo(true));
        assertThat(term1.toString(), equalTo(words.get(word)));
        ts.close();
    }
}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testDefaultsIcuAnalysis() {
    Settings settings = Settings.settingsBuilder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put("path.home", System.getProperty("path.home"))
            .build();

    Index index = new Index("test");

    Injector parentInjector = new ModulesBuilder()
            .add(new SettingsModule(settings),
                    new EnvironmentModule(new Environment(settings)))
            .createInjector();
    Injector injector = new ModulesBuilder().add(
            new IndexSettingsModule(index, settings),
            new IndexNameModule(index),
            new AnalysisModule(settings, parentInjector.getInstance(IndicesAnalysisService.class))
                    .addProcessor(new IcuAnalysisBinderProcessor()))
            .createChildInjector(parentInjector);

    AnalysisService analysisService = injector.getInstance(AnalysisService.class);

    TokenizerFactory tokenizerFactory = analysisService.tokenizer("icu_tokenizer");
    assertThat(tokenizerFactory, instanceOf(IcuTokenizerFactory.class));

    TokenFilterFactory filterFactory = analysisService.tokenFilter("icu_normalizer");
    assertThat(filterFactory, instanceOf(IcuNormalizerTokenFilterFactory.class));

    filterFactory = analysisService.tokenFilter("icu_folding");
    assertThat(filterFactory, instanceOf(IcuFoldingTokenFilterFactory.class));

    filterFactory = analysisService.tokenFilter("icu_transform");
    assertThat(filterFactory, instanceOf(IcuTransformTokenFilterFactory.class));

    Analyzer analyzer = analysisService.analyzer("icu_collation");
    assertThat(analyzer, instanceOf(NamedAnalyzer.class));

}
 

import org.elasticsearch.index.analysis.AnalysisService; //导入方法依赖的package包/类
@Test
public void testOne() throws IOException {

    String source = "Die Jahresfeier der Rechtsanwaltskanzleien auf dem Donaudampfschiff hat viel Ökosteuer gekostet";

    String[] expected = {
            "Die",
            "Die",
            "Jahresfeier",
            "Jahresfeier",
            "der",
            "der",
            "Rechtsanwaltskanzleien",
            "Rechtsanwaltskanzlei",
            "auf",
            "auf",
            "dem",
            "der",
            "Donaudampfschiff",
            "Donaudampfschiff",
            "hat",
            "haben",
            "viel",
            "viel",
            "Ökosteuer",
            "Ökosteuer",
            "gekostet",
            "kosten"
    };
    AnalysisService analysisService = MapperTestUtils.analysisService();
    TokenFilterFactory tokenFilter = analysisService.tokenFilter("baseform");
    Tokenizer tokenizer = analysisService.tokenizer("standard").create();
    tokenizer.setReader(new StringReader(source));
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}