Java TokenFilterFactory类代码示例

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
public synchronized TokenFilterFactory getTokenFilterFactory(final Version version) {
TokenFilterFactory factory = cache.get(version);
if (factory == null) {
    final String finalName = name().toLowerCase(Locale.ROOT);
    if (isMultiTermAware()) {
        factory = new MultiTermAwareTokenFilterFactory() {
            @Override
            public String name() {
                return finalName;
            }

            @Override
            public TokenStream create(TokenStream tokenStream) {
                return PreBuiltTokenFilters.this.create(tokenStream, version);
            }

            @Override
            public Object getMultiTermComponent() {
                return this;
            }
        };
    } else {
        factory = new TokenFilterFactory() {
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
private static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer)analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        final TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
            } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) {
                return (ShingleTokenFilterFactory.Factory) tokenFilterFactory;
            }
        }
    }
    return null;
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
private static boolean containsBrokenAnalysis(Analyzer analyzer) {
    // TODO maybe we need a getter on Namedanalyzer that tells if this uses broken Analysis
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer) analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof WordDelimiterTokenFilterFactory
                    || tokenFilterFactory instanceof EdgeNGramTokenFilterFactory) {
                return true;
            }
        }
    }
    return false;
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer)analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        final TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
            } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) {
                return (ShingleTokenFilterFactory.Factory) tokenFilterFactory;
            }
        }
    }
    return null;
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testTokenizerFilter() throws IOException {

    String source = "eins zwei drei";

    String[] expected = {
            "d800000000001214",
            "4a00000000000000",
            "c0000000000a100"
    };
    AnalysisService analysisService = MapperTestUtils.analysisService("eudex_analysis.json");
    TokenFilterFactory tokenFilterFactory = analysisService.tokenFilter("my_phonetic");
    Tokenizer tokenizer = analysisService.tokenizer("my_phonetic").create();
    tokenizer.setReader(new StringReader(source));
    assertSimpleTSOutput(tokenFilterFactory.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testOne() throws IOException {

    String source = "Das ist ein Bindestrich-Wort.";

    String[] expected = {
            "Das",
            "ist",
            "ein",
            "Bindestrich-Wort",
            "BindestrichWort",
            "Wort",
            "Bindestrich"
    };
    String resource = "org/xbib/elasticsearch/index/analysis/hyphen/hyphen_tokenizer.json";
    Tokenizer tokenizer = tokenizerFactory(resource, "my_hyphen_tokenizer").create();
    tokenizer.setReader(new StringReader(source));
    TokenFilterFactory tokenFilter = tokenFilterFactory(resource, "hyphen");
    TokenStream tokenStream = tokenFilter.create(tokenizer);
    assertSimpleTSOutput(tokenStream, expected);
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testTwo() throws IOException {

    String source = "Das E-Book muss dringend zum Buchbinder.";

    String[] expected = {
            "Das",
            "E-Book",
            "EBook",
            "Book",
            "muss",
            "dringend",
            "zum",
            "Buchbinder"
    };
    String resource = "org/xbib/elasticsearch/index/analysis/hyphen/hyphen_tokenizer.json";
    Tokenizer tokenizer = tokenizerFactory(resource, "my_hyphen_tokenizer").create();
    tokenizer.setReader(new StringReader(source));
    TokenFilterFactory tokenFilter = tokenFilterFactory(resource, "hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testFour() throws IOException {

    String source = "So wird's was: das Elasticsearch-Buch erscheint beim O'Reilly-Verlag.";

    String[] expected = {
            "So",
            "wird's",
            "was",
            "das",
            "Elasticsearch-Buch",
            "ElasticsearchBuch",
            "Buch",
            "Elasticsearch",
            "erscheint",
            "beim",
            "O'Reilly-Verlag"
    };
    String resource = "org/xbib/elasticsearch/index/analysis/hyphen/hyphen_tokenizer.json";
    Tokenizer tokenizer = tokenizerFactory(resource, "my_hyphen_tokenizer").create();
    tokenizer.setReader(new StringReader(source));
    TokenFilterFactory tokenFilter = tokenFilterFactory(resource, "hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testSix() throws IOException {

    String source = "E-Book";

    String[] expected = {
            "E-Book",
            "EBook",
            "Book"
    };

    String resource = "org/xbib/elasticsearch/index/analysis/hyphen/hyphen_tokenizer.json";
    Tokenizer tokenizer = tokenizerFactory(resource, "my_hyphen_tokenizer").create();
    tokenizer.setReader(new StringReader(source));
    TokenFilterFactory tokenFilter = tokenFilterFactory(resource, "hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testSeven() throws IOException {
    String source = "Procter & Gamble ist Procter&Gamble. Schwarz - weiss ist schwarz-weiss";

    String[] expected = {
            "Procter",
            "Gamble",
            "ist",
            "Procter&Gamble",
            "Schwarz",
            "weiss",
            "ist",
            "schwarz-weiss",
            "schwarzweiss",
            "weiss",
            "schwarz"
    };

    String resource = "org/xbib/elasticsearch/index/analysis/hyphen/hyphen_tokenizer.json";
    Tokenizer tokenizer = tokenizerFactory(resource, "my_hyphen_tokenizer").create();
    tokenizer.setReader(new StringReader(source));
    TokenFilterFactory tokenFilter = tokenFilterFactory(resource, "hyphen");
    assertSimpleTSOutput(tokenFilter.create(tokenizer), expected);
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
@Test
public void testOpenNLPAnalysis() throws Exception {
    AnalysisService analysisService = MapperTestUtils.analysisService(getResource());
    TokenizerFactory tokenizerFactory = analysisService.tokenizer("opennlp");
    MatcherAssert.assertThat(tokenizerFactory, instanceOf(OpenNLPTokenizerFactory.class));
    this.tokenizerFactory = (OpenNLPTokenizerFactory) tokenizerFactory;
    TokenFilterFactory filterFactory = analysisService.tokenFilter("opennlp");
    MatcherAssert.assertThat(filterFactory, instanceOf(OpenNLPTokenFilterFactory.class));
    this.filterFactory = (OpenNLPTokenFilterFactory) filterFactory;

    TokenFilterFactory filterpayloadsfilterFactory = analysisService.tokenFilter("filterpayloads");
    MatcherAssert.assertThat(filterpayloadsfilterFactory, instanceOf(FilterPayloadsTokenFilterFactory.class));
    TokenFilterFactory strippayloadsfilterFactory = analysisService.tokenFilter("strippayloads");
    MatcherAssert.assertThat(strippayloadsfilterFactory, instanceOf(StripPayloadsTokenFilterFactory.class));

    analyze();
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
public JapaneseBaseFormFilterFactory(final IndexSettings indexSettings, final Environment env, final String name,
        final Settings settings, final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    for (final String factoryClass : FACTORIES) {
        final Class<?> tokenFilterFactoryClass = fessAnalysisService.loadClass(factoryClass);
        if (tokenFilterFactoryClass != null) {
            if (logger.isInfoEnabled()) {
                logger.info("{} is found.", factoryClass);
            }
            tokenFilterFactory = AccessController.doPrivileged((PrivilegedAction<TokenFilterFactory>) () -> {
                try {
                    final Constructor<?> constructor = tokenFilterFactoryClass.getConstructor(IndexSettings.class, Environment.class,
                            String.class, Settings.class);
                    return (TokenFilterFactory) constructor.newInstance(indexSettings, env, name, settings);
                } catch (final Exception e) {
                    throw new ElasticsearchException("Failed to load " + factoryClass, e);
                }
            });
            break;
        } else if (logger.isDebugEnabled()) {
            logger.debug("{} is not found.", factoryClass);
        }
    }
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
public JapanesePartOfSpeechFilterFactory(final IndexSettings indexSettings, final Environment env, final String name,
        final Settings settings, final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    for (final String factoryClass : FACTORIES) {
        final Class<?> TokenFilterFactoryClass = fessAnalysisService.loadClass(factoryClass);
        if (TokenFilterFactoryClass != null) {
            if (logger.isInfoEnabled()) {
                logger.info("{} is found.", factoryClass);
            }
            tokenFilterFactory = AccessController.doPrivileged((PrivilegedAction<TokenFilterFactory>) () -> {
                try {
                    final Constructor<?> constructor = TokenFilterFactoryClass.getConstructor(IndexSettings.class, Environment.class,
                            String.class, Settings.class);
                    return (TokenFilterFactory) constructor.newInstance(indexSettings, env, name, settings);
                } catch (final Exception e) {
                    throw new ElasticsearchException("Failed to load " + factoryClass, e);
                }
            });
            break;
        } else if (logger.isDebugEnabled()) {
            logger.debug("{} is not found.", factoryClass);
        }
    }
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
public JapaneseKatakanaStemmerFactory(final IndexSettings indexSettings, final Environment env, final String name,
        final Settings settings, final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    for (final String factoryClass : FACTORIES) {
        final Class<?> tokenizerFactoryClass = fessAnalysisService.loadClass(factoryClass);
        if (tokenizerFactoryClass != null) {
            if (logger.isInfoEnabled()) {
                logger.info("{} is found.", factoryClass);
            }
            tokenFilterFactory = AccessController.doPrivileged((PrivilegedAction<TokenFilterFactory>) () -> {
                try {
                    final Constructor<?> constructor =
                            tokenizerFactoryClass.getConstructor(IndexSettings.class, Environment.class, String.class, Settings.class);
                    return (TokenFilterFactory) constructor.newInstance(indexSettings, env, name, settings);
                } catch (final Exception e) {
                    throw new ElasticsearchException("Failed to load " + factoryClass, e);
                }

            });
            break;
        } else if (logger.isDebugEnabled()) {
            logger.debug("{} is not found.", factoryClass);
        }
    }
}
 

import org.elasticsearch.index.analysis.TokenFilterFactory; //导入依赖的package包/类
public JapaneseReadingFormFilterFactory(final IndexSettings indexSettings, final Environment env, final String name,
        final Settings settings, final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    for (final String factoryClass : FACTORIES) {
        final Class<?> tokenFilterFactoryClass = fessAnalysisService.loadClass(factoryClass);
        if (tokenFilterFactoryClass != null) {
            if (logger.isInfoEnabled()) {
                logger.info("{} is found.", factoryClass);
            }
            tokenFilterFactory = AccessController.doPrivileged((PrivilegedAction<TokenFilterFactory>) () -> {
                try {
                    final Constructor<?> constructor = tokenFilterFactoryClass.getConstructor(IndexSettings.class, Environment.class,
                            String.class, Settings.class);
                    return (TokenFilterFactory) constructor.newInstance(indexSettings, env, name, settings);
                } catch (final Exception e) {
                    throw new ElasticsearchException("Failed to load " + factoryClass, e);
                }
            });
            break;
        } else if (logger.isDebugEnabled()) {
            logger.debug("{} is not found.", factoryClass);
        }
    }
}