Java NamedAnalyzer类代码示例

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
private List<TokenData> parse(String text) {
    NamedAnalyzer analyzer = getAnalysisService().indexAnalyzers.get("test");

    try {
        try (TokenStream ts = analyzer.tokenStream("test", new StringReader(text))) {
            List<TokenData> result = new ArrayList<>();
            CharTermAttribute charTerm = ts.addAttribute(CharTermAttribute.class);
            OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
            PositionIncrementAttribute position = ts.addAttribute(PositionIncrementAttribute.class);
            ts.reset();
            while (ts.incrementToken()) {
                String original = text.substring(offset.startOffset(), offset.endOffset());
                result.add(token(original, charTerm.toString(), position.getPositionIncrement()));
            }
            ts.end();

            return result;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
private static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer)analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        final TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
            } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) {
                return (ShingleTokenFilterFactory.Factory) tokenFilterFactory;
            }
        }
    }
    return null;
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
private static boolean containsBrokenAnalysis(Analyzer analyzer) {
    // TODO maybe we need a getter on Namedanalyzer that tells if this uses broken Analysis
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer) analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof WordDelimiterTokenFilterFactory
                    || tokenFilterFactory instanceof EdgeNGramTokenFilterFactory) {
                return true;
            }
        }
    }
    return false;
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    TokenCountFieldMapper.Builder builder = new TokenCountFieldMapper.Builder(name);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String propName = entry.getKey();
        Object propNode = entry.getValue();
        if (propName.equals("null_value")) {
            builder.nullValue(nodeIntegerValue(propNode));
            iterator.remove();
        } else if (propName.equals("analyzer")) {
            NamedAnalyzer analyzer = parserContext.getIndexAnalyzers().get(propNode.toString());
            if (analyzer == null) {
                throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
            }
            builder.analyzer(analyzer);
            iterator.remove();
        }
    }
    parseField(builder, name, node, parserContext);
    if (builder.analyzer() == null) {
        throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't.");
    }
    return builder;
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
@Override
public AllFieldMapper build(BuilderContext context) {
    // In case the mapping overrides these
    // TODO: this should be an exception! it doesnt make sense to not index this field
    if (fieldType.indexOptions() == IndexOptions.NONE) {
        fieldType.setIndexOptions(Defaults.FIELD_TYPE.indexOptions());
    } else {
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(),
            Defaults.POSITION_INCREMENT_GAP));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(),
            Defaults.POSITION_INCREMENT_GAP));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(),
            Defaults.POSITION_INCREMENT_GAP));
    }
    fieldType.setTokenized(true);

    return new AllFieldMapper(fieldType, enabled, context.indexSettings());
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
@Override
public TextFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
                + name + "] without positions enabled");
        }
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    setupFieldType(context);
    return new TextFieldMapper(
            name, fieldType, defaultFieldType, positionIncrementGap, includeInAll,
            context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
public void testVersionedAnalyzers() throws Exception {
    String yaml = "/org/elasticsearch/index/analysis/test1.yml";
    Settings settings2 = Settings.builder()
            .loadFromStream(yaml, getClass().getResourceAsStream(yaml))
            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
            .build();
    AnalysisRegistry newRegistry = getNewRegistry(settings2);
    IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings2);

    // registry always has the current version
    assertThat(newRegistry.getAnalyzer("default"), is(instanceOf(NamedAnalyzer.class)));
    NamedAnalyzer defaultNamedAnalyzer = (NamedAnalyzer) newRegistry.getAnalyzer("default");
    assertThat(defaultNamedAnalyzer.analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertEquals(Version.CURRENT.luceneVersion, defaultNamedAnalyzer.analyzer().getVersion());

    // analysis service has the expected version
    assertThat(indexAnalyzers.get("standard").analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertEquals(Version.V_2_0_0.luceneVersion, indexAnalyzers.get("standard").analyzer().getVersion());
    assertEquals(Version.V_2_0_0.luceneVersion, indexAnalyzers.get("thai").analyzer().getVersion());

    assertThat(indexAnalyzers.get("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), indexAnalyzers.get("custom7").analyzer().getVersion());
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
public void testAnalyzers() throws IOException {
    FakeFieldType fieldType1 = new FakeFieldType();
    fieldType1.setName("field1");
    fieldType1.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new FakeAnalyzer("index")));
    fieldType1.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new FakeAnalyzer("search")));
    fieldType1.setSearchQuoteAnalyzer(new NamedAnalyzer("baz", AnalyzerScope.INDEX, new FakeAnalyzer("search_quote")));
    FieldMapper fieldMapper1 = new FakeFieldMapper("field1", fieldType1);

    FakeFieldType fieldType2 = new FakeFieldType();
    fieldType2.setName("field2");
    FieldMapper fieldMapper2 = new FakeFieldMapper("field2", fieldType2);

    Analyzer defaultIndex = new FakeAnalyzer("default_index");
    Analyzer defaultSearch = new FakeAnalyzer("default_search");
    Analyzer defaultSearchQuote = new FakeAnalyzer("default_search_quote");

    DocumentFieldMappers documentFieldMappers = new DocumentFieldMappers(Arrays.asList(fieldMapper1, fieldMapper2), defaultIndex, defaultSearch, defaultSearchQuote);

    assertAnalyzes(documentFieldMappers.indexAnalyzer(), "field1", "index");
    assertAnalyzes(documentFieldMappers.searchAnalyzer(), "field1", "search");
    assertAnalyzes(documentFieldMappers.searchQuoteAnalyzer(), "field1", "search_quote");

    assertAnalyzes(documentFieldMappers.indexAnalyzer(), "field2", "default_index");
    assertAnalyzes(documentFieldMappers.searchAnalyzer(), "field2", "default_search");
    assertAnalyzes(documentFieldMappers.searchQuoteAnalyzer(), "field2", "default_search_quote");
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
public void testMergeSearchAnalyzer() throws Exception {
    DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
    String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type")
            .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").field("search_analyzer", "whitespace").endObject().endObject()
            .endObject().endObject().string();
    String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
            .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").field("search_analyzer", "keyword").endObject().endObject()
            .endObject().endObject().string();

    DocumentMapper existing = parser.parse("type", new CompressedXContent(mapping1));
    DocumentMapper changed = parser.parse("type", new CompressedXContent(mapping2));

    assertThat(((NamedAnalyzer) existing.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("whitespace"));
    DocumentMapper merged = existing.merge(changed.mapping(), false);

    assertThat(((NamedAnalyzer) merged.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("keyword"));
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
public void testChangeSearchAnalyzerToDefault() throws Exception {
    MapperService mapperService = createIndex("test").mapperService();
    String mapping1 = XContentFactory.jsonBuilder().startObject().startObject("type")
            .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").field("search_analyzer", "whitespace").endObject().endObject()
            .endObject().endObject().string();
    String mapping2 = XContentFactory.jsonBuilder().startObject().startObject("type")
            .startObject("properties").startObject("field").field("type", "text").field("analyzer", "standard").endObject().endObject()
            .endObject().endObject().string();

    DocumentMapper existing = mapperService.merge("type", new CompressedXContent(mapping1), MapperService.MergeReason.MAPPING_UPDATE, false);
    DocumentMapper merged = mapperService.merge("type", new CompressedXContent(mapping2), MapperService.MergeReason.MAPPING_UPDATE, false);

    assertThat(((NamedAnalyzer) existing.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("whitespace"));

    assertThat(((NamedAnalyzer) merged.mappers().getMapper("field").fieldType().searchAnalyzer()).name(), equalTo("standard"));
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
    Index index = new Index("_index", "testUUID");
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
    NamedAnalyzer namedAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
    IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, namedAnalyzer, namedAnalyzer, namedAnalyzer,
        Collections.emptyMap(), Collections.emptyMap());
    SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
    MapperService mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry(), similarityService,
        new IndicesModule(emptyList()).getMapperRegistry(), () -> null);
    XContentBuilder mappingSource = jsonBuilder().startObject().startObject("some_type")
        .startObject("properties")
        .endObject()
        .endObject().endObject();
    mapperService.merge("some_type", new CompressedXContent(mappingSource.string()), MergeReason.MAPPING_UPDATE, false);
    Set<String> allFields = new HashSet<>(mapperService.simpleMatchToIndexNames("*"));
    assertTrue(allFields.contains("_parent"));
    assertFalse(allFields.contains("_parent#null"));
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer)analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        final TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
            } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) {
                return (ShingleTokenFilterFactory.Factory) tokenFilterFactory;
            }
        }
    }
    return null;
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    TokenCountFieldMapper.Builder builder = tokenCountField(name);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String propName = Strings.toUnderscoreCase(entry.getKey());
        Object propNode = entry.getValue();
        if (propName.equals("null_value")) {
            builder.nullValue(nodeIntegerValue(propNode));
            iterator.remove();
        } else if (propName.equals("analyzer")) {
            NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
            if (analyzer == null) {
                throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
            }
            builder.analyzer(analyzer);
            iterator.remove();
        }
    }
    parseNumberField(builder, name, node, parserContext);
    if (builder.analyzer() == null) {
        throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't.");
    }
    return builder;
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
@Override
public LangStringFieldMapper build(final BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
                + name + "] without positions enabled");
        }
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    setupFieldType(context);
    return new LangStringFieldMapper(
            name, fieldType, defaultFieldType, positionIncrementGap, includeInAll,
            fieldSeparator, supportedLanguages, langField, langBaseName,
            context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
}
 

import org.elasticsearch.index.analysis.NamedAnalyzer; //导入依赖的package包/类
protected SherlokFieldMapper(final Names names, final FieldType fieldType,
        final Boolean docValues, final Boolean compress,
        final long compressThreshold,
        final PostingsFormatProvider postingsProvider,
        final DocValuesFormatProvider docValuesProvider,
        @Nullable final Settings fieldDataSettings,
        final MultiFields multiFields, final CopyTo copyTo,
        final NamedAnalyzer minhashAnalyzer, final String copyBitsTo) {
    super(names, 1.0f, fieldType, docValues, null, null, postingsProvider,
            docValuesProvider, null, null, fieldDataSettings, null,
            multiFields, copyTo);
    this.compress = compress;
    this.compressThreshold = compressThreshold;
    this.minhashAnalyzer = minhashAnalyzer;
    this.copyBitsTo = copyBitsTo;
}