Java BulkProcessor类代码示例

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public BulkProcessor buildBulkProcessor(Context context, TransportClient client) {
    bulkActions = context.getInteger(ES_BULK_ACTIONS,
            DEFAULT_ES_BULK_ACTIONS);
    bulkProcessorName = context.getString(ES_BULK_PROCESSOR_NAME,
            DEFAULT_ES_BULK_PROCESSOR_NAME);
    bulkSize = Util.getByteSizeValue(context.getInteger(ES_BULK_SIZE),
            context.getString(ES_BULK_SIZE_UNIT));
    concurrentRequest = context.getInteger(ES_CONCURRENT_REQUEST,
            DEFAULT_ES_CONCURRENT_REQUEST);
    flushIntervalTime = Util.getTimeValue(context.getString(ES_FLUSH_INTERVAL_TIME),
            DEFAULT_ES_FLUSH_INTERVAL_TIME);
    backoffPolicyTimeInterval = context.getString(ES_BACKOFF_POLICY_TIME_INTERVAL,
            DEFAULT_ES_BACKOFF_POLICY_START_DELAY);
    backoffPolicyRetries = context.getInteger(ES_BACKOFF_POLICY_RETRIES,
            DEFAULT_ES_BACKOFF_POLICY_RETRIES);
    return build(client);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
private BulkProcessor build(TransportClient client) {
    logger.trace("Bulk processor name: [{}]  bulkActions: [{}], bulkSize: [{}], flush interval time: [{}]," +
                    " concurrent Request: [{}], backoffPolicyTimeInterval: [{}], backoffPolicyRetries: [{}] ",
            new Object[]{bulkProcessorName, bulkActions, bulkSize, flushIntervalTime,
                    concurrentRequest, backoffPolicyTimeInterval, backoffPolicyRetries});
    return BulkProcessor.builder(client, getListener())
            .setName(bulkProcessorName)
            .setBulkActions(bulkActions)
            .setBulkSize(bulkSize)
            .setFlushInterval(flushIntervalTime)
            .setConcurrentRequests(concurrentRequest)
            .setBackoffPolicy(BackoffPolicy.exponentialBackoff(
                    Util.getTimeValue(backoffPolicyTimeInterval,
                            DEFAULT_ES_BACKOFF_POLICY_START_DELAY),
                    backoffPolicyRetries))
            .build();
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
@Override
public void configureBulkProcessorBackoff(
	BulkProcessor.Builder builder,
	@Nullable ElasticsearchSinkBase.BulkFlushBackoffPolicy flushBackoffPolicy) {

	BackoffPolicy backoffPolicy;
	if (flushBackoffPolicy != null) {
		switch (flushBackoffPolicy.getBackoffType()) {
			case CONSTANT:
				backoffPolicy = BackoffPolicy.constantBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
				break;
			case EXPONENTIAL:
			default:
				backoffPolicy = BackoffPolicy.exponentialBackoff(
					new TimeValue(flushBackoffPolicy.getDelayMillis()),
					flushBackoffPolicy.getMaxRetryCount());
		}
	} else {
		backoffPolicy = BackoffPolicy.noBackoff();
	}

	builder.setBackoffPolicy(backoffPolicy);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
private void execute(DocumentFactory<String> factory, InsertProperties properties, int from) throws InterruptedException {
	final int threads = properties.getNumOfThreads();
	ExecutorService service = Executors.newFixedThreadPool(threads);
	int perThread = properties.getDocPerIteration() / threads;
	int to = 0;
	Timer.Context insert = metrics.timer("insert-total").time();
	for(int i = 0; i < threads; i++) {
		BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkListener(metrics))
				.setBulkActions(properties.getBulkActions())
				.setBulkSize(new ByteSizeValue(1, ByteSizeUnit.GB))
				.setConcurrentRequests(properties.getBulkThreads())
				.build();
		from = to;
		to = from + perThread;
		SenderAction action = new SenderAction(metrics, properties, factory, bulkProcessor, from, to);
		service.execute(action);
	}
	service.shutdown();
	service.awaitTermination(60, TimeUnit.MINUTES);
	insert.stop();
	reporter.report();
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) {
  BulkProcessor.Builder builder = BulkProcessor.builder(client, listener);

  // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches.
  // This also means BulkProcessor#flush() is blocking as is also required.
  builder.setConcurrentRequests(0);

  if (config.getBulkFlushMaxActions().isPresent()) {
    builder.setBulkActions(config.getBulkFlushMaxActions().get());
  }
  if (config.getBulkFlushMaxSizeMB().isPresent()) {
    builder.setBulkSize(new ByteSizeValue(config.getBulkFlushMaxSizeMB().get(), ByteSizeUnit.MB));
  }
  if (config.getBulkFlushIntervalMS().isPresent()) {
    builder.setFlushInterval(TimeValue.timeValueMillis(config.getBulkFlushIntervalMS().get()));
  }

  return builder.build();
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  metrics = new ElasticsearchSystemProducerMetrics("es", new MetricsRegistryMap());
  producer = new ElasticsearchSystemProducer(SYSTEM_NAME,
                                             BULK_PROCESSOR_FACTORY,
                                             CLIENT,
                                             INDEX_REQUEST_FACTORY,
                                             metrics);

  processorOne = mock(BulkProcessor.class);
  processorTwo = mock(BulkProcessor.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), any(BulkProcessor.Listener.class)))
      .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), any(BulkProcessor.Listener.class)))
      .thenReturn(processorTwo);
  producer.register(SOURCE_TWO);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
@Test
public void testIgnoreVersionConficts() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
          ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
          .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  BulkResponse response = getRespWithFailedDocument(RestStatus.CONFLICT);

  listenerCaptor.getValue().afterBulk(0, null, response);
  assertEquals(1, metrics.conflicts.getCount());

  producer.flush(SOURCE_ONE);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf, "es."
            + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5));

    int bulkActions = ConfUtils.getInt(stormConf, "es." + boltType
            + ".bulkActions", 50);

    Client client = getClient(stormConf, boltType);

    BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(1).build();

    return new ElasticSearchConnection(client, bulkProcessor);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public static void flush(BulkProcessor bulkProcessor) {
    try {
        Field field = bulkProcessor.getClass().getDeclaredField("bulkRequest");
        if (field != null) {
            field.setAccessible(true);
            BulkRequest bulkRequest = (BulkRequest) field.get(bulkProcessor);
            if (bulkRequest.numberOfActions() > 0) {
                Method method = bulkProcessor.getClass().getDeclaredMethod("execute");
                if (method != null) {
                    method.setAccessible(true);
                    method.invoke(bulkProcessor);
                }
            }
        }
    } catch (Throwable e) {
        logger.error(e.getMessage(), e);
    }
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public void writeDataset(String indexName, String documentType, long numOfDocs, long numOfFields, BulkProcessor bulkProcessor){
  
  log.debug (String.format("*** Writing data to index %s", indexName));
  //TODO: create index with default schema
  if(this.templateBuilder!=null){
    Map<String, Object> mapping = new LinkedHashMap<String, Object> ();
    mapping.put("dynamic_templates", this.templateBuilder.build ());
    adminHelper.createIndex (indexName, documentType, mapping);
  }else{
    adminHelper.createIndex (indexName);
  }
  
  for(Integer i=0;i<numOfDocs;i++){
    IndexRequestBuilder indexRequestBuilder = client.prepareIndex (indexName, documentType, i.toString ())
            .setSource (generateRandomDoc (numOfFields));
    bulkProcessor.add (indexRequestBuilder.request ());
  } 
  bulkProcessor.flush ();    
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public void processDocumentActions(Stream<DocumentAction> documentActions)
{
	LOG.trace("Processing document actions ...");
	BulkProcessor bulkProcessor = bulkProcessorFactory.create(client);
	try
	{
		documentActions.forEachOrdered(documentAction ->
		{
			DocWriteRequest docWriteRequest = toDocWriteRequest(documentAction);
			bulkProcessor.add(docWriteRequest);
		});
	}
	finally
	{
		waitForCompletion(bulkProcessor);
		LOG.debug("Processed document actions.");
	}
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
private void waitForCompletion(BulkProcessor bulkProcessor)
{
	try
	{
		boolean isCompleted = bulkProcessor.awaitClose(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
		if (!isCompleted)
		{
			throw new MolgenisDataException("Failed to complete bulk request within the given time");
		}
	}
	catch (InterruptedException e)
	{
		Thread.currentThread().interrupt();
		throw new RuntimeException(e);
	}
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf, "es."
            + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5), "flushInterval");

    int bulkActions = ConfUtils.getInt(stormConf, "es." + boltType
            + ".bulkActions", 50);

    int concurrentRequests = ConfUtils.getInt(stormConf, "es." + boltType
            + ".concurrentRequests", 1);

    Client client = getClient(stormConf, boltType);

    BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(concurrentRequests).build();

    return new ElasticSearchConnection(client, bulkProcessor);
}
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
protected void dumpRestore(Client client, Builder builder) throws IOException {

        BulkProcessor bulkProcessor = buildBulkProcessor(client);
        BufferedReader reader = null;
        try {
            FileInputStream fis = new FileInputStream(builder.path());
            reader = new BufferedReader(new InputStreamReader(fis, builder.charset()));
            String line;
            while( (line=reader.readLine()) != null) {
                indexLine(bulkProcessor, builder.index(), builder.type(), line);
            }

        } finally {
            bulkProcessor.close();
            if (reader != null) {
                reader.close();
            }
        }
    }
 

import org.elasticsearch.action.bulk.BulkProcessor; //导入依赖的package包/类
private static ElasticConnection getConnection(String hostname, int transportPort, String flushIntervalString, BulkProcessor.Listener listener) {
    System.setProperty("es.set.netty.runtime.available.processors", "false");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString, TimeValue.timeValueSeconds(5), "flush");

    Client client = getClient(hostname, transportPort);

    BulkProcessor bulkProcessor = BulkProcessor.builder(client, listener)
            .setFlushInterval(flushInterval)
            .setBulkActions(10)
            .setConcurrentRequests(10)
            .build();

    return new ElasticConnection(client, bulkProcessor);
}