Java BulkProcessor.Listener方法代码示例

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) {
  BulkProcessor.Builder builder = BulkProcessor.builder(client, listener);

  // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches.
  // This also means BulkProcessor#flush() is blocking as is also required.
  builder.setConcurrentRequests(0);

  if (config.getBulkFlushMaxActions().isPresent()) {
    builder.setBulkActions(config.getBulkFlushMaxActions().get());
  }
  if (config.getBulkFlushMaxSizeMB().isPresent()) {
    builder.setBulkSize(new ByteSizeValue(config.getBulkFlushMaxSizeMB().get(), ByteSizeUnit.MB));
  }
  if (config.getBulkFlushIntervalMS().isPresent()) {
    builder.setFlushInterval(TimeValue.timeValueMillis(config.getBulkFlushIntervalMS().get()));
  }

  return builder.build();
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
@Test
public void testIgnoreVersionConficts() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
          ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
          .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  BulkResponse response = getRespWithFailedDocument(RestStatus.CONFLICT);

  listenerCaptor.getValue().afterBulk(0, null, response);
  assertEquals(1, metrics.conflicts.getCount());

  producer.flush(SOURCE_ONE);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf, "es."
            + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5));

    int bulkActions = ConfUtils.getInt(stormConf, "es." + boltType
            + ".bulkActions", 50);

    Client client = getClient(stormConf, boltType);

    BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(1).build();

    return new ElasticSearchConnection(client, bulkProcessor);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf, "es."
            + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5), "flushInterval");

    int bulkActions = ConfUtils.getInt(stormConf, "es." + boltType
            + ".bulkActions", 50);

    int concurrentRequests = ConfUtils.getInt(stormConf, "es." + boltType
            + ".concurrentRequests", 1);

    Client client = getClient(stormConf, boltType);

    BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(concurrentRequests).build();

    return new ElasticSearchConnection(client, bulkProcessor);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
private static ElasticConnection getConnection(String hostname, int transportPort, String flushIntervalString, BulkProcessor.Listener listener) {
    System.setProperty("es.set.netty.runtime.available.processors", "false");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString, TimeValue.timeValueSeconds(5), "flush");

    Client client = getClient(hostname, transportPort);

    BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
            .setFlushInterval(flushInterval)
            .setBulkActions(10)
            .setConcurrentRequests(10)
            .build();

    return new ElasticConnection(client, bulkProcessor);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
@Test
public void no_value_inserted_when_not_enough_requests() {

    // Create Mocks:
    Client mockedTransportClient = mock(Client.class);
    BulkProcessor.Listener mockedBulkProcessorListener = mock(BulkProcessor.Listener.class);

    // Configure the BulkProcessor to use:
    BulkProcessorConfiguration configuration = new BulkProcessorConfiguration(new BulkProcessingOptionsBuilder().build(), mockedBulkProcessorListener);

    // And create a fake index builder:
    IndexRequestBuilder indexRequestBuilder = new IndexRequestBuilder(mockedTransportClient, IndexAction.INSTANCE);

    // The mapping to use:
    IElasticSearchMapping localWeatherDataMapper = new LocalWeatherDataMapper();

    // Index to insert to:
    String indexName = "weather_data";

    // Initialize it with the default settings:
    when(mockedTransportClient.settings())
            .thenReturn(Settings.builder().build());

    when(mockedTransportClient.prepareIndex())
            .thenReturn(indexRequestBuilder);

    // Create the Test subject:
    ElasticSearchClient<LocalWeatherData> elasticSearchClient = new ElasticSearchClient<>(mockedTransportClient, indexName, localWeatherDataMapper, configuration);

    // Create more entities, than Bulk insertion threshold:
    Stream<LocalWeatherData> entitiesStream = getData(configuration.getBulkProcessingOptions().getBulkActions() - 1).stream();

    // Index the Data:
    elasticSearchClient.index(entitiesStream);

    // Verify, that the TransportClient bulk insert has been called:
    verify(mockedTransportClient, times(0)).bulk(anyObject(), anyObject());
    verify(mockedBulkProcessorListener, times(0)).beforeBulk(anyLong(), anyObject());
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
/**
 * Build the {@link BulkProcessor}.
 *
 * <p>Note: this is exposed for testing purposes.
 */
@VisibleForTesting
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
	checkNotNull(listener);

	BulkProcessor.Builder bulkProcessorBuilder = BulkProcessor.builder(client, listener);

	// This makes flush() blocking
	bulkProcessorBuilder.setConcurrentRequests(0);

	if (bulkProcessorFlushMaxActions != null) {
		bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
	}

	if (bulkProcessorFlushMaxSizeMb != null) {
		bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
	}

	if (bulkProcessorFlushIntervalMillis != null) {
		bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
	}

	// if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
	callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);

	return bulkProcessorBuilder.build();
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
@Test(expected=SamzaException.class)
public void testFlushFailedSendFromException() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
      ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
      .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  listenerCaptor.getValue().afterBulk(0, null, new Throwable());

  producer.flush(SOURCE_ONE);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
@Test(expected=SamzaException.class)
public void testFlushFailedSendFromFailedDocument() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
      ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
      .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  BulkResponse response = getRespWithFailedDocument(RestStatus.BAD_REQUEST);

  listenerCaptor.getValue().afterBulk(0, null, response);

  producer.flush(SOURCE_ONE);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
public BulkProcessorConfiguration(BulkProcessingOptions options, BulkProcessor.Listener listener) {
    this.options = options;
    this.listener = listener;
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入方法依赖的package包/类
public BulkProcessor.Listener getBulkProcessorListener() {
    return listener;
}