PHP Client::bulk方法代码示例

 /**
  * Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing.
  *
  * (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.)
  *
  * @throws PartialFailureException
  */
 public function flush()
 {
     // We might have nothing to do
     if ($this->getItemCount() == 0) {
         return;
     }
     $this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount()));
     $params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList];
     $result = $this->elasticSearch->bulk($params);
     if (count($result['items']) != $this->itemCount) {
         $expected = $this->itemCount;
         $actual = count($result['items']);
         throw new PartialFailureException("The wrong number of items was stored; expected {$expected}, but stored {$actual}");
     }
     if (boolval($result['errors'])) {
         $errorList = [];
         foreach ($result['items'] as $item) {
             if ($item['index']['status'] < 200 || $item['index']['status'] >= 300) {
                 $errorList[] = $item;
             }
         }
         throw new PartialFailureException("Some items failed. " . json_encode($errorList));
     }
     $this->clear();
 }

 /**
  * {@inheritdoc}
  */
 protected function setUp()
 {
     parent::setUp();
     $this->client = ClientBuilder::create()->build();
     $this->deleteIndex();
     $this->client->indices()->create(array_filter(['index' => self::INDEX_NAME, 'mapping' => $this->getMapping()]));
     $bulkBody = [];
     foreach ($this->getDataArray() as $type => $documents) {
         foreach ($documents as $id => $document) {
             $bulkBody[] = ['index' => ['_index' => self::INDEX_NAME, '_type' => $type, '_id' => $id]];
             $bulkBody[] = $document;
         }
     }
     $this->client->bulk(['body' => $bulkBody]);
     $this->client->indices()->refresh();
 }

 /**
  * Flushes the current query container to the index, used for bulk queries execution.
  */
 public function commit()
 {
     $this->isReadOnly('Commit');
     $this->bulkQueries = array_merge($this->bulkQueries, $this->bulkParams);
     $this->client->bulk($this->bulkQueries);
     $this->flush();
     $this->bulkQueries = [];
 }

 public function it_adds_multiple_searchable_objects_to_the_search_index(Client $elasticsearch, Searchable $searchableObject)
 {
     $searchableObjects = [$searchableObject];
     $params = [];
     $params['body'][] = ['index' => ['_id' => $this->searchableId, '_index' => $this->indexName, '_type' => $this->searchableType]];
     $params['body'][] = $this->searchableBody;
     $elasticsearch->bulk($params)->shouldBeCalled();
     $this->upsertToIndex($searchableObjects);
 }

 /**
  * Insert an array of documents.
  *
  * @param array $docs
  */
 public function insertDocuments(array $docs)
 {
     $params = [];
     while (!empty($docs)) {
         $doc = array_shift($docs);
         $params['body'][] = ['index' => ['_index' => env('ES_INDEX'), '_type' => $this->type, '_id' => uniqid()]];
         $params['body'][] = $doc;
         $results = $this->client->bulk($params);
     }
 }

 /**
  * @param ShopIndex $index
  * @param int[] $ids
  */
 public function index(ShopIndex $index, $ids)
 {
     if (empty($ids)) {
         return;
     }
     $blog = $this->provider->get($ids);
     $remove = array_diff($ids, array_keys($blog));
     $documents = [];
     foreach ($blog as $row) {
         $documents[] = ['index' => ['_id' => $row->getId()]];
         $documents[] = $row;
     }
     foreach ($remove as $id) {
         $documents[] = ['delete' => ['_id' => $id]];
     }
     if (empty($documents)) {
         return;
     }
     $this->client->bulk(['index' => $index->getName(), 'type' => 'blog', 'body' => $documents]);
 }

 private function copyData(Client $client, $source_index, $destination_index, $batch_size)
 {
     $search_params = array("search_type" => "scan", "scroll" => "30s", "size" => 50, "index" => $source_index, "body" => array("query" => array("match_all" => array())));
     $search_responses = new SearchResponseIterator($client, $search_params);
     foreach ($search_responses as $search_response) {
         $index_params = array();
         $index_params['index'] = $destination_index;
         foreach ($search_response['hits']['hits'] as $hit) {
             $index_params['body'][] = array('type' => $hit['_type'], 'id' => $hit['_id']);
             $index_params['body'][] = $hit['source'];
         }
         $client->bulk($index_params);
     }
 }

 /**
  * Flushes the items in this Bulk object to Elasticsearch. If there are no items, this method does nothing.
  *
  * (In particular, note that if no items are in the Bulk object, the bulk flush event is not fired.)
  *
  * @throws PartialFailureException
  */
 public function flush()
 {
     // We might have nothing to do
     if ($this->getItemCount() == 0) {
         return;
     }
     $this->dispatchEvent(Events::BULK_FLUSH, new BulkFlushEvent($this->index, $this->type, $this->getItemCount()));
     $params = ['index' => $this->index, 'type' => $this->type, 'body' => $this->itemList];
     $result = $this->elasticSearch->bulk($params);
     if (!empty($result->errors)) {
         throw new PartialFailureException(count($result->errors) . " items failed.", $result->errors);
     }
     $this->itemList = [];
     $this->itemCount = 0;
 }

 /**
  * @param string $type entity name
  * @param array $data [[id => body]]
  * @return array
  *
  * @see http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html
  */
 public function addToIndexBulk($type, array $data)
 {
     $queries = [];
     foreach ($data as $id => $body) {
         $queries[] = ['index' => ['_id' => $id]];
         $queries[] = $body;
     }
     $res = parent::bulk(['index' => $this->index, 'type' => $type, 'body' => $queries]);
     foreach ($res['items'] as $queries) {
         foreach ($queries as $query) {
             if (isset($query['error'])) {
                 throw new ElasticsearchException($query['error']);
             }
         }
     }
     return $res;
 }

 /**
  * @param ShopIndex $index
  * @param string[] $numbers
  * @return \string[]
  */
 public function indexProducts(ShopIndex $index, $numbers)
 {
     if (empty($numbers)) {
         return;
     }
     $products = $this->provider->get($index->getShop(), $numbers);
     $remove = array_diff($numbers, array_keys($products));
     $documents = [];
     foreach ($products as $product) {
         $documents[] = ['index' => ['_id' => $product->getNumber()]];
         $documents[] = $product;
     }
     foreach ($remove as $number) {
         $documents[] = ['delete' => ['_id' => $number]];
     }
     $this->client->bulk(['index' => $index->getName(), 'type' => ProductMapping::TYPE, 'body' => $documents]);
 }

 /**
  * {@inheritDoc}
  */
 public function executeBulk(BulkRequestInterface $bulk)
 {
     $bulkParams = ['body' => $bulk->getOperations()];
     $rawBulkResponse = $this->client->bulk($bulkParams);
     /**
      * @var BulkResponseInterface
      */
     $bulkResponse = $this->objectManager->create('Smile\\ElasticsuiteCore\\Api\\Index\\Bulk\\BulkResponseInterface', ['rawResponse' => $rawBulkResponse]);
     if ($bulkResponse->hasErrors()) {
         foreach ($bulkResponse->aggregateErrorsByReason() as $error) {
             $sampleDocumentIds = implode(', ', array_slice($error['document_ids'], 0, 10));
             $errorMessages = [sprintf("Bulk %s operation failed %d times in index %s for type %s", $error['operation'], $error['count'], $error['index'], $error['document_type']), sprintf("Error (%s) : %s", $error['error']['type'], $error['error']['reason']), sprintf("Failed doc ids sample : %s", $sampleDocumentIds)];
             $this->logger->error(implode(" ", $errorMessages));
         }
     }
     return $bulkResponse;
 }

 /**
  * @param ShopIndex $index
  * @param int[] $groupIds
  */
 public function indexProperties(ShopIndex $index, $groupIds)
 {
     if (empty($groupIds)) {
         return;
     }
     /** @var Group[] $properties */
     $properties = $this->provider->get($index->getShop(), $groupIds);
     $remove = array_diff($groupIds, array_keys($properties));
     $documents = [];
     foreach ($properties as $property) {
         $documents[] = ['index' => ['_id' => $property->getId()]];
         $documents[] = $property;
     }
     foreach ($remove as $id) {
         $documents[] = ['delete' => ['_id' => $id]];
     }
     $this->client->bulk(['index' => $index->getName(), 'type' => PropertyMapping::TYPE, 'body' => $documents]);
 }

 /**
  * Inserts the current query container to the index, used for bulk queries execution.
  *
  * @param array $params Parameters that will be passed to the flush or refresh queries.
  *
  * @return null|array
  */
 public function commit(array $params = [])
 {
     $this->isReadOnly('Commit');
     if (!empty($this->bulkQueries)) {
         $bulkQueries = array_merge($this->bulkQueries, $this->bulkParams);
         $this->bulkQueries = [];
         $bulkResponse = $this->client->bulk($bulkQueries);
         switch ($this->getCommitMode()) {
             case 'flush':
                 $this->flush($params);
                 break;
             case 'refresh':
             default:
                 $this->refresh($params);
                 break;
         }
         return $bulkResponse;
     }
     return null;
 }

 /**
  * @param CsvFile $file
  * @param LoadOptions $options
  * @param $primaryIndex
  * @return bool
  */
 public function loadFile(CsvFile $file, LoadOptions $options, $primaryIndex = null)
 {
     $csvHeader = $file->getHeader();
     $params = ['body' => []];
     $iBulk = 1;
     foreach ($file as $i => $line) {
         // skip header
         if (!$i) {
             continue;
         }
         $lineData = array_combine($csvHeader, $line);
         if ($primaryIndex) {
             if (!array_key_exists($primaryIndex, $lineData)) {
                 $this->logger->error(sprintf("CSV error: Missing id column %s on line %s", $primaryIndex, $i + 1));
                 return false;
             }
             $params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType(), '_id' => $lineData[$primaryIndex]]];
         } else {
             $params['body'][] = ['index' => ['_index' => $options->getIndex(), '_type' => $options->getType()]];
         }
         $params['body'][] = $lineData;
         if ($i % $options->getBulkSize() == 0) {
             $this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
             $responses = $this->client->bulk($params);
             $this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
             $params = ['body' => []];
             if ($responses['errors'] !== false) {
                 if (!empty($responses['items'])) {
                     foreach ($responses['items'] as $itemResult) {
                         if (!empty($itemResult['index']['error'])) {
                             if (is_array($itemResult['index']['error'])) {
                                 $this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
                             } else {
                                 $this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
                             }
                             return false;
                         }
                     }
                 }
                 return false;
             }
             $iBulk++;
             unset($responses);
         }
     }
     if (!empty($params['body'])) {
         $this->logger->info(sprintf("Write %s batch %d to %s start", $options->getType(), $iBulk, $options->getIndex()));
         $responses = $this->client->bulk($params);
         $this->logger->info(sprintf("Write %s batch %d to %s took %d ms", $options->getType(), $iBulk, $options->getIndex(), $responses['took']));
         if ($responses['errors'] !== false) {
             if (!empty($responses['items'])) {
                 foreach ($responses['items'] as $itemResult) {
                     if (!empty($itemResult['index']['error'])) {
                         if (is_array($itemResult['index']['error'])) {
                             $this->logger->error(sprintf("ES error: %s", $this->getErrorMessageFromErrorField($itemResult['index']['error'])));
                         } else {
                             $this->logger->error(sprintf("ES error: %s", $itemResult['index']['error']));
                         }
                         return false;
                     }
                 }
             }
             return false;
         }
         unset($responses);
     }
     return true;
 }

<?php

include __DIR__ . '/../../vendor/autoload.php';
use Utils\ZillowClient;
use Elasticsearch\Client;
$client = new Client(['hosts' => ['http://localhost:9200']]);
$zClient = new ZillowClient();
$results = json_decode($zClient->getDataForAllZips());
$params = [];
$params['index'] = 'hrqls';
$params['type'] = 'houseData';
foreach ($results as $item) {
    $params['body'][] = array('create' => array('_id' => sha1($item->location->lat . $item->location->lon)));
    $params['body'][] = array('state' => $item->state, 'city' => $item->city, 'zip' => $item->zip, 'location' => array('lat' => $item->location->lat, 'lon' => $item->location->lon), 'avgHomeValueIndex' => $item->homeData->avgHomeValueIndex, 'avgHomesRecentlySold' => $item->homeData->avgHomesRecentlySold, 'avgPropertyTax' => $item->homeData->avgPropertyTax, 'turnoverWithinLastYear' => $item->homeData->turnoverWithinLastYear);
}
$client->bulk($params);