Java TokenizerFactory类代码示例

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public synchronized TokenizerFactory getTokenizerFactory(final Version version) {
TokenizerFactory tokenizerFactory = cache.get(version);
if (tokenizerFactory == null) {
    final String finalName = name().toLowerCase(Locale.ROOT);
    if (getMultiTermComponent(version) != null) {
        tokenizerFactory = new MultiTermAwareTokenizerFactory() {
            @Override
            public String name() {
                return finalName;
            }

            @Override
            public Tokenizer create() {
                return PreBuiltTokenizers.this.create(version);
            }

            @Override
            public Object getMultiTermComponent() {
                return PreBuiltTokenizers.this.getMultiTermComponent(version);
            }
        };
    } else {
        tokenizerFactory = new TokenizerFactory() {
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
private NamedRegistry<AnalysisProvider<TokenizerFactory>> setupTokenizers(List<AnalysisPlugin> plugins) {
    NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = new NamedRegistry<>("tokenizer");
    tokenizers.register("standard", StandardTokenizerFactory::new);
    tokenizers.register("uax_url_email", UAX29URLEmailTokenizerFactory::new);
    tokenizers.register("path_hierarchy", PathHierarchyTokenizerFactory::new);
    tokenizers.register("PathHierarchy", PathHierarchyTokenizerFactory::new);
    tokenizers.register("keyword", KeywordTokenizerFactory::new);
    tokenizers.register("letter", LetterTokenizerFactory::new);
    tokenizers.register("lowercase", LowerCaseTokenizerFactory::new);
    tokenizers.register("whitespace", WhitespaceTokenizerFactory::new);
    tokenizers.register("nGram", NGramTokenizerFactory::new);
    tokenizers.register("ngram", NGramTokenizerFactory::new);
    tokenizers.register("edgeNGram", EdgeNGramTokenizerFactory::new);
    tokenizers.register("edge_ngram", EdgeNGramTokenizerFactory::new);
    tokenizers.register("pattern", PatternTokenizerFactory::new);
    tokenizers.register("classic", ClassicTokenizerFactory::new);
    tokenizers.register("thai", ThaiTokenizerFactory::new);
    tokenizers.extractAndRegister(plugins, AnalysisPlugin::getTokenizers);
    return tokenizers;
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
@Test
public void testOpenNLPAnalysis() throws Exception {
    AnalysisService analysisService = MapperTestUtils.analysisService(getResource());
    TokenizerFactory tokenizerFactory = analysisService.tokenizer("opennlp");
    MatcherAssert.assertThat(tokenizerFactory, instanceOf(OpenNLPTokenizerFactory.class));
    this.tokenizerFactory = (OpenNLPTokenizerFactory) tokenizerFactory;
    TokenFilterFactory filterFactory = analysisService.tokenFilter("opennlp");
    MatcherAssert.assertThat(filterFactory, instanceOf(OpenNLPTokenFilterFactory.class));
    this.filterFactory = (OpenNLPTokenFilterFactory) filterFactory;

    TokenFilterFactory filterpayloadsfilterFactory = analysisService.tokenFilter("filterpayloads");
    MatcherAssert.assertThat(filterpayloadsfilterFactory, instanceOf(FilterPayloadsTokenFilterFactory.class));
    TokenFilterFactory strippayloadsfilterFactory = analysisService.tokenFilter("strippayloads");
    MatcherAssert.assertThat(strippayloadsfilterFactory, instanceOf(StripPayloadsTokenFilterFactory.class));

    analyze();
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public ReloadableJapaneseTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name,
        final Settings settings, final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    for (final String factoryClass : FACTORIES) {
        final Class<?> tokenizerFactoryClass = fessAnalysisService.loadClass(factoryClass);
        if (tokenizerFactoryClass != null) {
            if (logger.isInfoEnabled()) {
                logger.info("{} is found.", factoryClass);
            }
            tokenizerFactory = AccessController.doPrivileged((PrivilegedAction<TokenizerFactory>) () -> {
                try {
                    final Constructor<?> constructor =
                            tokenizerFactoryClass.getConstructor(IndexSettings.class, Environment.class, String.class, Settings.class);
                    return (TokenizerFactory) constructor.newInstance(indexSettings, env, name, settings);
                } catch (final Exception e) {
                    throw new ElasticsearchException("Failed to load " + factoryClass, e);
                }
            });
            break;
        } else if (logger.isDebugEnabled()) {
            logger.debug("{} is not found.", factoryClass);
        }
    }
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public ChineseTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name, final Settings settings,
        final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    final Class<?> tokenizerFactoryClass = fessAnalysisService.loadClass(FACTORY);
    if (tokenizerFactoryClass != null) {
        if (logger.isInfoEnabled()) {
            logger.info("{} is found.", FACTORY);
        }
        tokenizerFactory = AccessController.doPrivileged((PrivilegedAction<TokenizerFactory>) () -> {
            try {
                final Constructor<?> constructor =
                        tokenizerFactoryClass.getConstructor(IndexSettings.class, Environment.class, String.class, Settings.class);
                return (TokenizerFactory) constructor.newInstance(indexSettings, env, name, settings);
            } catch (final Exception e) {
                throw new ElasticsearchException("Failed to load " + FACTORY, e);
            }
        });
    } else if (logger.isDebugEnabled()) {
        logger.debug("{} is not found.", FACTORY);
    }
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public VietnameseTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name, final Settings settings,
        final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    final Class<?> tokenizerFactoryClass = fessAnalysisService.loadClass(FACTORY);
    if (tokenizerFactoryClass != null) {
        if (logger.isInfoEnabled()) {
            logger.info("{} is found.", FACTORY);
        }
        tokenizerFactory = AccessController.doPrivileged((PrivilegedAction<TokenizerFactory>) () -> {
            try {
                final Constructor<?> constructor =
                        tokenizerFactoryClass.getConstructor(IndexSettings.class, Environment.class, String.class, Settings.class);
                return (TokenizerFactory) constructor.newInstance(indexSettings, env, name, settings);
            } catch (final Exception e) {
                throw new ElasticsearchException("Failed to load " + FACTORY, e);
            }
        });
    } else if (logger.isDebugEnabled()) {
        logger.debug("{} is not found.", FACTORY);
    }
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public JapaneseTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name, final Settings settings,
        final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    for (final String factoryClass : FACTORIES) {
        final Class<?> tokenizerFactoryClass = fessAnalysisService.loadClass(factoryClass);
        if (tokenizerFactoryClass != null) {
            if (logger.isInfoEnabled()) {
                logger.info("{} is found.", factoryClass);
            }
            tokenizerFactory = AccessController.doPrivileged((PrivilegedAction<TokenizerFactory>) () -> {
                try {
                    final Constructor<?> constructor =
                            tokenizerFactoryClass.getConstructor(IndexSettings.class, Environment.class, String.class, Settings.class);
                    return (TokenizerFactory) constructor.newInstance(indexSettings, env, name, settings);
                } catch (final Exception e) {
                    throw new ElasticsearchException("Failed to load " + factoryClass, e);
                }
            });
            break;
        } else if (logger.isDebugEnabled()) {
            logger.debug("{} is not found.", factoryClass);
        }
    }
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public KoreanTokenizerFactory(final IndexSettings indexSettings, final Environment env, final String name, final Settings settings,
        final FessAnalysisService fessAnalysisService) {
    super(indexSettings, name, settings);

    final Class<?> tokenizerFactoryClass = fessAnalysisService.loadClass(FACTORY);
    if (tokenizerFactoryClass != null) {
        if (logger.isInfoEnabled()) {
            logger.info("{} is found.", FACTORY);
        }
        tokenizerFactory = AccessController.doPrivileged((PrivilegedAction<TokenizerFactory>) () -> {
            try {
                final Constructor<?> constructor =
                        tokenizerFactoryClass.getConstructor(IndexSettings.class, Environment.class, String.class, Settings.class);
                return (TokenizerFactory) constructor.newInstance(indexSettings, env, name, settings);
            } catch (final Exception e) {
                throw new ElasticsearchException("Failed to load " + FACTORY, e);
            }
        });
    } else if (logger.isDebugEnabled()) {
        logger.debug("{} is not found.", FACTORY);
    }
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
@Override
public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
    final Map<String, AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();
    extra.put("fess_japanese_tokenizer", (indexSettings, env, name, settings) -> new JapaneseTokenizerFactory(indexSettings, env, name,
            settings, pluginComponent.getFessAnalysisService()));
    extra.put("fess_japanese_reloadable_tokenizer",
            (indexSettings, env, name, settings) -> new ReloadableJapaneseTokenizerFactory(indexSettings, env, name, settings,
                    pluginComponent.getFessAnalysisService()));
    extra.put("fess_korean_tokenizer", (indexSettings, env, name, settings) -> new KoreanTokenizerFactory(indexSettings, env, name,
            settings, pluginComponent.getFessAnalysisService()));
    extra.put("fess_vietnamese_tokenizer", (indexSettings, env, name, settings) -> new VietnameseTokenizerFactory(indexSettings, env,
            name, settings, pluginComponent.getFessAnalysisService()));
    extra.put("fess_simplified_chinese_tokenizer", (indexSettings, env, name, settings) -> new ChineseTokenizerFactory(indexSettings,
            env, name, settings, pluginComponent.getFessAnalysisService()));
    return extra;
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
@Test
public void testDefaultsIcuAnalysis() throws IOException {

    CharFilterFactory charFilterFactory = charFilterFactory("icu_normalizer");
    assertThat(charFilterFactory, instanceOf(IcuNormalizerCharFilterFactory.class));

    TokenizerFactory tf = tokenizerFactory("icu_tokenizer");
    assertThat(tf, instanceOf(IcuTokenizerFactory.class));

    TokenFilterFactory filterFactory = tokenFilterFactory("icu_normalizer");
    assertThat(filterFactory, instanceOf(IcuNormalizerTokenFilterFactory.class));

    filterFactory = tokenFilterFactory("icu_folding");
    assertThat(filterFactory, instanceOf(IcuFoldingTokenFilterFactory.class));

    filterFactory = tokenFilterFactory("icu_transform");
    assertThat(filterFactory, instanceOf(IcuTransformTokenFilterFactory.class));

    Analyzer analyzer = MapperTestUtils.analyzer( "icu_collation");
    assertThat(analyzer, instanceOf(NamedAnalyzer.class));
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
@Override
public Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> getTokenizers() {
  Map<String, AnalysisModule.AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();

  extra.put("jieba_search", JiebaTokenizerFactory::getJiebaSearchTokenizerFactory);
  extra.put("jieba_index", JiebaTokenizerFactory::getJiebaIndexTokenizerFactory);

  return extra;
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public TestAnalysis(IndexAnalyzers indexAnalyzers,
                    Map<String, TokenFilterFactory> tokenFilter,
                    Map<String, TokenizerFactory> tokenizer,
                    Map<String, CharFilterFactory> charFilter) {
    this.indexAnalyzers = indexAnalyzers;
    this.tokenFilter = tokenFilter;
    this.tokenizer = tokenizer;
    this.charFilter = charFilter;
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
public AnalysisModule(Environment environment, List<AnalysisPlugin> plugins) throws IOException {
    NamedRegistry<AnalysisProvider<CharFilterFactory>> charFilters = setupCharFilters(plugins);
    NamedRegistry<org.apache.lucene.analysis.hunspell.Dictionary> hunspellDictionaries = setupHunspellDictionaries(plugins);
    hunspellService = new HunspellService(environment.settings(), environment, hunspellDictionaries.getRegistry());
    NamedRegistry<AnalysisProvider<TokenFilterFactory>> tokenFilters = setupTokenFilters(plugins, hunspellService);
    NamedRegistry<AnalysisProvider<TokenizerFactory>> tokenizers = setupTokenizers(plugins);
    NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> analyzers = setupAnalyzers(plugins);
    NamedRegistry<AnalysisProvider<AnalyzerProvider<?>>> normalizers = setupNormalizers(plugins);
    analysisRegistry = new AnalysisRegistry(environment, charFilters.getRegistry(), tokenFilters.getRegistry(), tokenizers
        .getRegistry(), analyzers.getRegistry(), normalizers.getRegistry());
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
private static TokenStream createStackedTokenStream(String source, CharFilterFactory[] charFilterFactories, TokenizerFactory tokenizerFactory, TokenFilterFactory[] tokenFilterFactories, int current) {
    Reader reader = new FastStringReader(source);
    for (CharFilterFactory charFilterFactory : charFilterFactories) {
        reader = charFilterFactory.create(reader);
    }
    Tokenizer tokenizer = tokenizerFactory.create();
    tokenizer.setReader(reader);
    TokenStream tokenStream = tokenizer;
    for (int i = 0; i < current; i++) {
        tokenStream = tokenFilterFactories[i].create(tokenStream);
    }
    return tokenStream;
}
 

import org.elasticsearch.index.analysis.TokenizerFactory; //导入依赖的package包/类
@Override
public Map<String, AnalysisProvider<TokenizerFactory>> getTokenizers() {
    Map<String, AnalysisProvider<TokenizerFactory>> extra = new HashMap<>();
    extra.put("smartcn_tokenizer", SmartChineseTokenizerTokenizerFactory::new);
    // This is an alias to "smartcn_tokenizer"; it's here for backwards compat
    extra.put("smartcn_sentence", SmartChineseTokenizerTokenizerFactory::new);
    return extra;
}