Java IndicesFieldDataCache类代码示例

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
/**
 * This method verifies that the given {@code metaData} holds sane values to create an {@link IndexService}.
 * This method tries to update the meta data of the created {@link IndexService} if the given {@code metaDataUpdate} is different from the given {@code metaData}.
 * This method will throw an exception if the creation or the update fails.
 * The created {@link IndexService} will not be registered and will be closed immediately.
 */
public synchronized void verifyIndexMetadata(IndexMetaData metaData, IndexMetaData metaDataUpdate) throws IOException {
    final List<Closeable> closeables = new ArrayList<>();
    try {
        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() {});
        closeables.add(indicesFieldDataCache);
        IndicesQueryCache indicesQueryCache = new IndicesQueryCache(settings);
        closeables.add(indicesQueryCache);
        // this will also fail if some plugin fails etc. which is nice since we can verify that early
        final IndexService service =
            createIndexService("metadata verification", metaData, indicesQueryCache, indicesFieldDataCache, emptyList(), s -> {});
        closeables.add(() -> service.close("metadata verification", false));
        service.mapperService().merge(metaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        if (metaData.equals(metaDataUpdate) == false) {
            service.updateMetaData(metaDataUpdate);
        }
    } finally {
        IOUtils.close(closeables);
    }
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
protected <A extends Aggregator, B extends AggregationBuilder> A createAggregator(B aggregationBuilder,
                                                                                  IndexSearcher indexSearcher,
                                                                                  MappedFieldType... fieldTypes) throws IOException {
    IndexSettings indexSettings = createIndexSettings();
    SearchContext searchContext = createSearchContext(indexSearcher, indexSettings);
    CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
    when(searchContext.bigArrays()).thenReturn(new MockBigArrays(Settings.EMPTY, circuitBreakerService));
    // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
    MapperService mapperService = mapperServiceMock();
    when(mapperService.hasNested()).thenReturn(false);
    when(searchContext.mapperService()).thenReturn(mapperService);
    IndexFieldDataService ifds = new IndexFieldDataService(indexSettings,
            new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() {
            }), circuitBreakerService, mapperService);
    when(searchContext.fieldData()).thenReturn(ifds);

    SearchLookup searchLookup = new SearchLookup(mapperService, ifds, new String[]{"type"});
    when(searchContext.lookup()).thenReturn(searchLookup);

    QueryShardContext queryShardContext = queryShardContextMock(fieldTypes, indexSettings, circuitBreakerService);
    when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);

    @SuppressWarnings("unchecked")
    A aggregator = (A) aggregationBuilder.build(searchContext, null).create(null, true);
    return aggregator;
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
private void doTestRequireDocValues(MappedFieldType ft) {
    ThreadPool threadPool = new TestThreadPool("random_threadpool_name");
    try {
        IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null);
        IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null);
        ft.setName("some_long");
        ft.setHasDocValues(true);
        ifds.getForField(ft); // no exception
        ft.setHasDocValues(false);
        try {
            ifds.getForField(ft);
            fail();
        } catch (IllegalArgumentException e) {
            assertThat(e.getMessage(), containsString("doc values"));
        }
    } finally {
        threadPool.shutdown();
    }
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
@Override
protected void configure() {
    bindQueryParsersExtension();
    bindHunspellExtension();
    bindMapperExtension();

    bind(IndicesLifecycle.class).to(InternalIndicesLifecycle.class).asEagerSingleton();
    bind(IndicesService.class).asEagerSingleton();
    bind(RecoverySettings.class).asEagerSingleton();
    bind(RecoveryTarget.class).asEagerSingleton();
    bind(RecoverySource.class).asEagerSingleton();
    bind(IndicesStore.class).asEagerSingleton();
    bind(IndicesClusterStateService.class).asEagerSingleton();
    bind(IndexingMemoryController.class).asEagerSingleton();
    bind(SyncedFlushService.class).asEagerSingleton();
    bind(IndicesQueryCache.class).asEagerSingleton();
    bind(IndicesRequestCache.class).asEagerSingleton();
    bind(IndicesFieldDataCache.class).asEagerSingleton();
    bind(TransportNodesListShardStoreMetaData.class).asEagerSingleton();
    bind(IndicesTTLService.class).asEagerSingleton();
    bind(IndicesWarmer.class).asEagerSingleton();
    bind(UpdateHelper.class).asEagerSingleton();
    bind(MetaDataIndexUpgradeService.class).asEagerSingleton();
    bind(IndicesFieldDataCacheListener.class).asEagerSingleton();
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
/**
 * creates a new initializing shard.
 *
 * @param routing              shard routing to use
 * @param shardPath            path to use for shard data
 * @param indexMetaData        indexMetaData for the shard, including any mapping
 * @param indexSearcherWrapper an optional wrapper to be used during searchers
 * @param listeners            an optional set of listeners to add to the shard
 */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,
                              @Nullable IndexSearcherWrapper indexSearcherWrapper,
                              Runnable globalCheckpointSyncer,
                              @Nullable EngineFactory engineFactory,
                              IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
    final IndexShard indexShard;
    final Store store = createStore(indexSettings, shardPath);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
                indexSettings.getSettings());
        mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
        final IndexEventListener indexEventListener = new IndexEventListener() {
        };
        final Engine.Warmer warmer = searcher -> {
        };
        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
        });
        IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, indicesFieldDataCache,
            new NoneCircuitBreakerService(), mapperService);
        indexShard = new IndexShard(routing, indexSettings, shardPath, store, indexCache, mapperService, similarityService,
            indexFieldDataService, engineFactory, indexEventListener, indexSearcherWrapper, threadPool,
            BigArrays.NON_RECYCLING_INSTANCE, warmer, globalCheckpointSyncer, Collections.emptyList(), Arrays.asList(listeners));
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
/**
 * This creates a new IndexService without registering it
 */
private synchronized IndexService createIndexService(final String reason,
                                                     IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache,
                                                     IndicesFieldDataCache indicesFieldDataCache,
                                                     List<IndexEventListener> builtInListeners,
                                                     Consumer<ShardId> globalCheckpointSyncer,
                                                     IndexingOperationListener... indexingOperationListeners) throws IOException {
    final Index index = indexMetaData.getIndex();
    final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopeSetting);
    logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]",
        indexMetaData.getIndex(),
        idxSettings.getNumberOfShards(),
        idxSettings.getNumberOfReplicas(),
        idxSettings.isShadowReplicaIndex() ? "s" : "", reason);

    final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry);
    for (IndexingOperationListener operationListener : indexingOperationListeners) {
        indexModule.addIndexOperationListener(operationListener);
    }
    pluginsService.onIndexModule(indexModule);
    for (IndexEventListener listener : builtInListeners) {
        indexModule.addIndexEventListener(listener);
    }
    return indexModule.newIndexService(
        nodeEnv,
        xContentRegistry,
        this,
        circuitBreakerService,
        bigArrays,
        threadPool,
        scriptService,
        clusterService,
        client,
        indicesQueryCache,
        mapperRegistry,
        globalCheckpointSyncer,
        indicesFieldDataCache);
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
CacheCleaner(IndicesFieldDataCache cache, IndicesRequestCache requestCache, Logger logger, ThreadPool threadPool, TimeValue interval) {
    this.cache = cache;
    this.requestCache = requestCache;
    this.logger = logger;
    this.threadPool = threadPool;
    this.interval = interval;
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
public IndexFieldDataService(IndexSettings indexSettings, IndicesFieldDataCache indicesFieldDataCache,
                             CircuitBreakerService circuitBreakerService, MapperService mapperService) {
    super(indexSettings);
    this.indicesFieldDataCache = indicesFieldDataCache;
    this.circuitBreakerService = circuitBreakerService;
    this.mapperService = mapperService;
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
@Inject
public IndexFieldDataService(Index index, IndexSettingsService indexSettingsService, IndicesFieldDataCache indicesFieldDataCache,
                             CircuitBreakerService circuitBreakerService, MapperService mapperService) {
    super(index, indexSettingsService.getSettings());
    this.indicesFieldDataCache = indicesFieldDataCache;
    this.circuitBreakerService = circuitBreakerService;
    this.mapperService = mapperService;
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
@Override
public void ensureEstimatedStats() {
    if (size() > 0) {
        // Checks that the breakers have been reset without incurring a
        // network request, because a network request can increment one
        // of the breakers
        for (NodeAndClient nodeAndClient : nodes.values()) {
            final IndicesFieldDataCache fdCache = getInstanceFromNode(IndicesService.class, nodeAndClient.node).getIndicesFieldDataCache();
            // Clean up the cache, ensuring that entries' listeners have been called
            fdCache.getCache().refresh();

            final String name = nodeAndClient.name;
            final CircuitBreakerService breakerService = getInstanceFromNode(CircuitBreakerService.class, nodeAndClient.node);
            CircuitBreaker fdBreaker = breakerService.getBreaker(CircuitBreaker.FIELDDATA);
            assertThat("Fielddata breaker not reset to 0 on node: " + name, fdBreaker.getUsed(), equalTo(0L));
            // Anything that uses transport or HTTP can increase the
            // request breaker (because they use bigarrays), because of
            // that the breaker can sometimes be incremented from ping
            // requests from other clusters because Jenkins is running
            // multiple ES testing jobs in parallel on the same machine.
            // To combat this we check whether the breaker has reached 0
            // in an assertBusy loop, so it will try for 10 seconds and
            // fail if it never reached 0
            try {
                assertBusy(new Runnable() {
                    @Override
                    public void run() {
                        CircuitBreaker reqBreaker = breakerService.getBreaker(CircuitBreaker.REQUEST);
                        assertThat("Request breaker not reset to 0 on node: " + name, reqBreaker.getUsed(), equalTo(0L));
                    }
                });
            } catch (Exception e) {
                fail("Exception during check for request breaker reset to 0: " + e);
            }

            NodeService nodeService = getInstanceFromNode(NodeService.class, nodeAndClient.node);
            CommonStatsFlags flags = new CommonStatsFlags(Flag.FieldData, Flag.QueryCache, Flag.Segments);
            NodeStats stats = nodeService.stats(flags, false, false, false, false, false, false, false, false, false, false, false);
            assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
            assertThat("Query cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
            assertThat("FixedBitSet cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getSegments().getBitsetMemoryInBytes(), equalTo(0L));
        }
    }
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
public IndicesFieldDataCache getIndicesFieldDataCache() {
    return indicesFieldDataCache;
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
public IndexService newIndexService(
    NodeEnvironment environment,
    NamedXContentRegistry xContentRegistry,
    IndexService.ShardStoreDeleter shardStoreDeleter,
    CircuitBreakerService circuitBreakerService,
    BigArrays bigArrays,
    ThreadPool threadPool,
    ScriptService scriptService,
    ClusterService clusterService,
    Client client,
    IndicesQueryCache indicesQueryCache,
    MapperRegistry mapperRegistry,
    Consumer<ShardId> globalCheckpointSyncer,
    IndicesFieldDataCache indicesFieldDataCache)
    throws IOException {
    final IndexEventListener eventListener = freeze();
    IndexSearcherWrapperFactory searcherWrapperFactory = indexSearcherWrapper.get() == null
        ? (shard) -> null : indexSearcherWrapper.get();
    eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
    final String storeType = indexSettings.getValue(INDEX_STORE_TYPE_SETTING);
    final IndexStore store;
    if (Strings.isEmpty(storeType) || isBuiltinType(storeType)) {
        store = new IndexStore(indexSettings);
    } else {
        Function<IndexSettings, IndexStore> factory = storeTypes.get(storeType);
        if (factory == null) {
            throw new IllegalArgumentException("Unknown store type [" + storeType + "]");
        }
        store = factory.apply(indexSettings);
        if (store == null) {
            throw new IllegalStateException("store must not be null");
        }
    }
    final QueryCache queryCache;
    if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
        BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
        if (queryCacheProvider == null) {
            queryCache = new IndexQueryCache(indexSettings, indicesQueryCache);
        } else {
            queryCache = queryCacheProvider.apply(indexSettings, indicesQueryCache);
        }
    } else {
        queryCache = new DisabledQueryCache(indexSettings);
    }
    return new IndexService(indexSettings, environment, xContentRegistry, new SimilarityService(indexSettings, similarities),
            shardStoreDeleter, analysisRegistry, engineFactory.get(), circuitBreakerService, bigArrays, threadPool, scriptService,
            clusterService, client, queryCache, store, eventListener, searcherWrapperFactory, mapperRegistry,
            indicesFieldDataCache, globalCheckpointSyncer, searchOperationListeners, indexOperationListeners);
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
public IndexService(IndexSettings indexSettings, NodeEnvironment nodeEnv,
                    NamedXContentRegistry xContentRegistry,
                    SimilarityService similarityService,
                    ShardStoreDeleter shardStoreDeleter,
                    AnalysisRegistry registry,
                    @Nullable EngineFactory engineFactory,
                    CircuitBreakerService circuitBreakerService,
                    BigArrays bigArrays,
                    ThreadPool threadPool,
                    ScriptService scriptService,
                    ClusterService clusterService,
                    Client client,
                    QueryCache queryCache,
                    IndexStore indexStore,
                    IndexEventListener eventListener,
                    IndexModule.IndexSearcherWrapperFactory wrapperFactory,
                    MapperRegistry mapperRegistry,
                    IndicesFieldDataCache indicesFieldDataCache,
                    Consumer<ShardId> globalCheckpointSyncer,
                    List<SearchOperationListener> searchOperationListeners,
                    List<IndexingOperationListener> indexingOperationListeners) throws IOException {
    super(indexSettings);
    this.indexSettings = indexSettings;
    this.globalCheckpointSyncer = globalCheckpointSyncer;
    this.xContentRegistry = xContentRegistry;
    this.similarityService = similarityService;
    this.mapperService = new MapperService(indexSettings, registry.build(indexSettings), xContentRegistry, similarityService,
        mapperRegistry,
        // we parse all percolator queries as they would be parsed on shard 0
        () -> newQueryShardContext(0, null, () -> {
            throw new IllegalArgumentException("Percolator queries are not allowed to use the current timestamp");
        }));
    this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService);
    this.shardStoreDeleter = shardStoreDeleter;
    this.bigArrays = bigArrays;
    this.threadPool = threadPool;
    this.scriptService = scriptService;
    this.clusterService = clusterService;
    this.client = client;
    this.eventListener = eventListener;
    this.nodeEnv = nodeEnv;
    this.indexStore = indexStore;
    indexFieldData.setListener(new FieldDataCacheListener(this));
    this.bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetCacheListener(this));
    this.warmer = new IndexWarmer(indexSettings.getSettings(), threadPool,
        bitsetFilterCache.createListener(threadPool));
    this.indexCache = new IndexCache(indexSettings, queryCache, bitsetFilterCache);
    this.engineFactory = engineFactory;
    // initialize this last -- otherwise if the wrapper requires any other member to be non-null we fail with an NPE
    this.searcherWrapper = wrapperFactory.newWrapper(this);
    this.globalCheckpointTask = new AsyncGlobalCheckpointTask(this);
    this.indexingOperationListeners = Collections.unmodifiableList(indexingOperationListeners);
    this.searchOperationListeners = Collections.unmodifiableList(searchOperationListeners);
    // kick off async ops for the first shard in this index
    this.refreshTask = new AsyncRefreshTask(this);
    rescheduleFsyncTask(indexSettings.getTranslogDurability());
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
private IndexService newIndexService(IndexModule module) throws IOException {
    return module.newIndexService(nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool,
            scriptService, clusterService, null, indicesQueryCache, mapperRegistry, shardId -> {},
            new IndicesFieldDataCache(settings, listener));
}
 

import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; //导入依赖的package包/类
public void testIndicesFieldDataCacheSetting() {
    assertMemorySizeSetting(IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY, "indices.fielddata.cache.size",
            new ByteSizeValue(-1));
}