Java ConcurrentUpdateSolrClient类代码示例

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public void doFeed() throws Exception {
    this.server = new ConcurrentUpdateSolrClient(serverUrl,queueSize,threads);
    if(threads==1 && queueSize == 1) {
        this.server = new HttpSolrClient(serverUrl);
    }

    if(deleteIndex) {
        System.out.println("Delete the index.");
        server.deleteByQuery("*:*");
    }

    parse();
    server.commit();
    server.optimize();
    server.close();
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
/**
 * Performs a batch index of _many_ documents, uses the ConcurrentUpdateSolrClient.
 * <p>If you want faster indexing, disable the autoCommit and autoSoftCommit functionalities,
 * see {@link #disableAutoCommit()}.
 * @param source of document to index.
 * @throws IOException network error.
 * @throws SolrServerException solr error.
 */
public void batchIndex(Iterator<SolrInputDocument> source) throws SolrServerException, IOException
{
   String dhus_url = configurationManager.getServerConfiguration().getUrl() + SOLR_SVC;
   try (ConcurrentUpdateSolrClient client = new ConcurrentUpdateSolrClient(dhus_url, 1000, 10))
   {
      client.add(source);
      client.commit(true, true);
   }
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
@MCRCommand(
    syntax = "create solr metadata and content index at {0}",
    help = "create solr's metadata and content index on specific solr server core",
    order = 120)
public static void createIndex(String url) throws Exception {
    MCRSolrCore core = new MCRSolrCore(url);
    SolrClient concurrentSolrClient = core.getConcurrentClient();
    SolrClient solrClient = core.getClient();
    MCRSolrIndexer.rebuildMetadataIndex(concurrentSolrClient);
    MCRSolrIndexer.rebuildContentIndex(solrClient);
    if (concurrentSolrClient instanceof ConcurrentUpdateSolrClient) {
        ((ConcurrentUpdateSolrClient) concurrentSolrClient).blockUntilFinished();
    }
    solrClient.optimize();
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
@Override
public void index() throws IOException, SolrServerException {
    if (documents == null || documents.isEmpty()) {
        LOGGER.warn("No input documents to index.");
        return;
    }
    int totalCount = documents.size();
    LOGGER.info("Handling {} documents", totalCount);
    SolrClient solrClient = getSolrClient();
    if (solrClient instanceof ConcurrentUpdateSolrClient) {
        LOGGER.info("Detected ConcurrentUpdateSolrClient. Split up batch update.");
        splitDocuments();
        //for statistics:
        documents.clear();
        return;
    }
    UpdateResponse updateResponse;
    try {
        UpdateRequest updateRequest = getUpdateRequest(MCRSolrConstants.UPDATE_PATH);
        updateRequest.add(documents);
        updateResponse = updateRequest.process(getSolrClient());
    } catch (Throwable e) {
        LOGGER.warn("Error while indexing document collection. Split and retry.");
        splitDocuments();
        return;
    }
    if (updateResponse.getStatus() != 0) {
        LOGGER.error("Error while indexing document collection. Split and retry: {}", updateResponse.getResponse());
        splitDocuments();
    } else {
        LOGGER.info("Sending {} documents was successful in {} ms.", totalCount, updateResponse.getElapsedTime());
    }
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
@Override
public void stop() {
  for (ConcurrentUpdateSolrClient client : this.clients.values()) {
    try {
      client.shutdownNow();
      client.blockUntilFinished();
      client.close();
    } catch (Exception e) {
      log.error("Exception thrown while closing client.", e);
    }
  }
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public static SolrClient getClient(Map stormConf, String boltType) {
    String zkHost = ConfUtils.getString(stormConf, "solr." + boltType
            + ".zkhost", null);
    String solrUrl = ConfUtils.getString(stormConf, "solr." + boltType
            + ".url", null);
    String collection = ConfUtils.getString(stormConf, "solr." + boltType
            + ".collection", null);
    int queueSize = ConfUtils.getInt(stormConf, "solr." + boltType
            + ".queueSize", -1);

    SolrClient client;

    if (StringUtils.isNotBlank(zkHost)) {
        client = new CloudSolrClient.Builder().withZkHost(zkHost).build();
        if (StringUtils.isNotBlank(collection)) {
            ((CloudSolrClient) client).setDefaultCollection(collection);
        }
    } else if (StringUtils.isNotBlank(solrUrl)) {
        if (queueSize == -1) {
            client = new HttpSolrClient.Builder(solrUrl).build();
        } else {
            client = new ConcurrentUpdateSolrClient.Builder(solrUrl)
                    .withQueueSize(queueSize).build();
        }
    } else {
        throw new RuntimeException(
                "SolrClient should have zk or solr URL set up");
    }

    return client;
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
protected void setup(String serverURL, String name) {
    if (!serverURL.endsWith("/")) {
        serverURL += serverURL + "/";
    }
    this.serverURL = serverURL;
    this.name = name;
    String coreURL = serverURL + name;
    int connectionTimeout = MCRConfiguration.instance().getInt(CONFIG_PREFIX + "SolrClient.ConnectionTimeout");
    int socketTimeout = MCRConfiguration.instance().getInt(CONFIG_PREFIX + "SolrClient.SocketTimeout");

    // default server
    solrClient = new HttpSolrClient.Builder(coreURL).build();
    solrClient.setRequestWriter(new BinaryRequestWriter());
    solrClient.setConnectionTimeout(connectionTimeout);
    solrClient.setSoTimeout(socketTimeout);
    // concurrent server
    if (USE_CONCURRENT_SERVER) {
        int queueSize = MCRConfiguration.instance().getInt(CONFIG_PREFIX + "ConcurrentUpdateSolrClient.QueueSize");
        int threadCount = MCRConfiguration.instance()
            .getInt(CONFIG_PREFIX + "ConcurrentUpdateSolrClient.ThreadCount");
        concurrentClient = new ConcurrentUpdateSolrClient.Builder(coreURL)
            .withQueueSize(queueSize)
            .withThreadCount(threadCount)
            .build();
        concurrentClient.setRequestWriter(new BinaryRequestWriter());
        concurrentClient.setConnectionTimeout(connectionTimeout);
        concurrentClient.setSoTimeout(socketTimeout);
    }
    // shutdown handler
    MCRShutdownHandler.getInstance().addCloseable(new MCRShutdownHandler.Closeable() {

        @Override
        public void prepareClose() {
        }

        @Override
        public int getPriority() {
            return Integer.MIN_VALUE + 5;
        }

        @Override
        public void close() {
            shutdown();
        }
    });
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
@Override
public void index() throws IOException, SolrServerException {
    int totalCount = contentMap.size();
    LOGGER.info("Handling {} documents", totalCount);
    //multithread processing will result in too many http request
    UpdateResponse updateResponse;
    try {
        Iterator<SolrInputDocument> documents = MCRSolrInputDocumentFactory.getInstance().getDocuments(contentMap);
        SolrClient solrClient = getSolrClient();
        if (solrClient instanceof ConcurrentUpdateSolrClient) {
            //split up to speed up processing
            splitup(documents);
            return;
        }
        if (LOGGER.isDebugEnabled()) {
            ArrayList<SolrInputDocument> debugList = new ArrayList<>();
            while (documents.hasNext()) {
                debugList.add(documents.next());
            }
            LOGGER.debug("Sending these documents: {}", debugList);
            //recreate documents interator;
            documents = debugList.iterator();
        }
        if (solrClient instanceof HttpSolrClient) {
            updateResponse = solrClient.add(documents);
        } else {
            ArrayList<SolrInputDocument> docs = new ArrayList<>(totalCount);
            while (documents.hasNext()) {
                docs.add(documents.next());
            }
            updateResponse = solrClient.add(docs);
        }
    } catch (Throwable e) {
        LOGGER.warn("Error while indexing document collection. Split and retry.", e);
        splitup();
        return;
    }
    if (updateResponse.getStatus() != 0) {
        LOGGER.error("Error while indexing document collection. Split and retry: {}", updateResponse.getResponse());
        splitup();
    } else {
        LOGGER.info("Sending {} documents was successful in {} ms.", totalCount, updateResponse.getElapsedTime());
    }

}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public static void main(String[] args) throws CorruptIndexException, IOException, SolrServerException {

        if (args.length < 3) {
            System.err.println("Usage: java -Dfile.encoding=UTF8 -Dclient.encoding.override=UTF-8 -Xmx256m -Xms256m -server " + IndexLoader.class.getName()
                    + " </path/to/index> <AutoCompleteSolrUrl> <indexField1,acField1> [indexField2,acField2 ... ]");
            System.exit(0);
        }
        Map<String,String> fieldMap = getFieldMapping(args, 2);
        DirectoryReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(args[0])));
        int docs = reader.maxDoc();
        SolrClient solr = new ConcurrentUpdateSolrClient.Builder(args[1]).withQueueSize(10000).withThreadCount(2).build();
        Set<SolrInputDocument> batch = new HashSet<SolrInputDocument>(1000);
        
        Bits liveDocs = MultiFields.getLiveDocs(reader);
        
        // go through all docs in the index
        for (int i = 0; i < docs; i++) {
            // process doc only if not deleted
            if (liveDocs == null || liveDocs.get(i)) {
                // loop through all fields to be looked at
                SolrInputDocument doc = new SolrInputDocument();
                Iterator<String> iter = fieldMap.keySet().iterator();
                
                boolean phraseFieldEmpty = false;
                
                while (iter.hasNext()) {
                    String indexField = iter.next();
                    String acField = fieldMap.get(indexField);
                    IndexableField field = reader.document(i).getField(indexField);
                    String value = field != null ? reader.document(i).getField(indexField).stringValue() : null;
                    
                    if (field != null && value != null && !value.isEmpty()) {
                      doc.addField(acField, value);
                    } else {
                      // not very relevant piece of info
                      // System.err.println("Field is null or empty, skipping: " + indexField);
                      
                      if (acField.equalsIgnoreCase("phrase")) {
                        System.err.println("Since AC phrase field would be null, this doc will not be created: " + reader.document(i));
                        phraseFieldEmpty = true;
                        break;
                      }
                    }
                }

                if (!phraseFieldEmpty) {
                  solr.add(doc);
                  if (docs % 1000 == 0) {
                    System.out.println("Docs: " + docs);
                  }
                }
            }
        }
        if (!batch.isEmpty())
            solr.add(batch);
        reader.close();
        System.out.println("Optimizing...");
        solr.optimize();
        solr.close();
    }
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public FileLoader(String solrURL) throws MalformedURLException {
    solr = new ConcurrentUpdateSolrClient.Builder(solrURL).withQueueSize(10000).withThreadCount(2).build();
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public ConcurrentUpdateSolrClient getUpdateSolrServer() {
    return updateSolrServer;
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public void setUpdateSolrServer(ConcurrentUpdateSolrClient updateSolrServer) {
    this.updateSolrServer = updateSolrServer;
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
@Override
public Producer createProducer() throws Exception {
    // do we have servers?
    SolrComponent.SolrServerReference ref = getComponent().getSolrServers(this);
    if (ref == null) {
        // no then create new servers
        ref = new SolrComponent.SolrServerReference();
        CloudSolrClient cloudServer = getCloudServer();
        if (cloudServer == null) {
            HttpSolrClient solrServer = new HttpSolrClient(url);
            ConcurrentUpdateSolrClient solrStreamingServer = new ConcurrentUpdateSolrClient(url, streamingQueueSize, streamingThreadCount);

            // set the properties on the solr server
            if (maxRetries != null) {
                solrServer.setMaxRetries(maxRetries);
            }
            if (soTimeout != null) {
                solrServer.setSoTimeout(soTimeout);
            }
            if (connectionTimeout != null) {
                solrServer.setConnectionTimeout(connectionTimeout);
            }
            if (defaultMaxConnectionsPerHost != null) {
                solrServer.setDefaultMaxConnectionsPerHost(defaultMaxConnectionsPerHost);
            }
            if (maxTotalConnections != null) {
                solrServer.setMaxTotalConnections(maxTotalConnections);
            }
            if (followRedirects != null) {
                solrServer.setFollowRedirects(followRedirects);
            }
            if (allowCompression != null) {
                solrServer.setAllowCompression(allowCompression);
            }
            ref.setSolrServer(solrServer);
            ref.setUpdateSolrServer(solrStreamingServer);
        }
        ref.setCloudSolrServer(cloudServer);

        getComponent().addSolrServers(this, ref);
    }

    ref.addReference();
    return new SolrProducer(this, ref.getSolrServer(), ref.getUpdateSolrServer(), ref.getCloudSolrServer());
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
@Override
public RecordReader<IntWritable, LWDocumentWritable>  getRecordReader(
    InputSplit split, JobConf job, Reporter reporter) throws IOException {

  LWInputSplit lwSplit = (LWInputSplit) split;
  SolrClient solr;
  String collection = job.get(SOLR_COLLECTION);
  String query = job.get(SOLR_QUERY, "*:*");
  if (lwSplit.isZk) {
    SolrSecurity.setSecurityConfig(job);
    if (lwSplit.isShard) {
      solr = new HttpSolrClient.Builder()
          .withBaseSolrUrl(lwSplit.getConnectionUri())
          .build();
    } else {
      // somehow the list of shard is unavailable
      solr = new CloudSolrClient.Builder()
          .withZkHost(lwSplit.getConnectionUri())
          .build();
      ((CloudSolrClient) solr).setDefaultCollection(collection);
    }

  } else {
    String connectionUri = lwSplit.getConnectionUri();
    int queueSize = job.getInt("solr.client.queue.size", 100);
    int threadCount = job.getInt("solr.client.threads", 1);

    if (!connectionUri.endsWith("/")) {
      connectionUri += "/";
    }

    connectionUri += collection;
    solr = new ConcurrentUpdateSolrClient.Builder(connectionUri)
        .withQueueSize(queueSize)
        .withThreadCount(threadCount)
        .build();
  }

  SolrQuery solrQuery = new SolrQuery(query);
  solrQuery.set("distrib", false);
  solrQuery.set("collection", collection);
  if (solrQuery.getRows() == null) {
    solrQuery.setRows(DEFAULT_PAGE_SIZE); // default page size
  }
  solrQuery.setSort(SolrQuery.SortClause.asc("id"));

  return new LWRecordReader(solr, solrQuery, "*");
}
 

import org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient; //导入依赖的package包/类
public void open(Configuration job, String name) throws IOException {
  this.name = name;
  log.info("Opening LucidWorksWriter for {}", name);
  //If the solr server has not already been set, then set it from configs.
  if (solr == null) {
    SolrSecurity.setSecurityConfig(job);
    String zkHost = job.get(SOLR_ZKHOST);
    String collection = job.get(SOLR_COLLECTION, "collection1");
    if (zkHost != null && !zkHost.equals("")) {
      log.info("Indexing to collection: " + collection + " w/ ZK host: " + zkHost);
      solr = new CloudSolrClient.Builder()
          .withZkHost(zkHost)
          .build();
      ((CloudSolrClient) solr).setDefaultCollection(collection);
      ((CloudSolrClient) solr).connect();
    } else {
      String solrURL = job.get(SOLR_SERVER_URL);
      if (!solrURL.endsWith("/")) {
        solrURL += "/";
      }
      solrURL += collection;
      int queueSize = job.getInt("solr.client.queue.size", 100);
      int threadCount = job.getInt("solr.client.threads", 1);
      solr = new ConcurrentUpdateSolrClient.Builder(solrURL)
          .withQueueSize(queueSize)
          .withThreadCount(threadCount)
          .build();
    }
  }
  String paramsString = job.get("solr.params");
  if (paramsString != null) {
    params = new ModifiableSolrParams();
    String[] pars = paramsString.trim().split("\\&");
    for (String kvs : pars) {
      String[] kv = kvs.split("=");
      if (kv.length < 2) {
        log.warn("Invalid Solr param " + kvs + ", skipping...");
        continue;
      }
      params.add(kv[0], kv[1]);
    }
    log.info("Using Solr params: " + params.toString());
  }
  includeMetadata = job.getBoolean("lw.metadata", false);
  includeAnnotations = job.getBoolean("lw.annotations", false);
  popMappingsFromAnnotsTypesAndFeats(job);
  //Should we support buffering when writing?
  bufferSize = job.getInt("lww.buffer.docs.size", 500);
  if (bufferSize > 0) {
    buffer = new ArrayList<SolrInputDocument>(bufferSize);
  } else {
    buffer = new ArrayList<SolrInputDocument>();
  }
  commitOnClose = job.getBoolean("lww.commit.on.close", false);
  maxRetries = job.getInt("lww.max.retries", 3);
  sleep = job.getInt("lww.retry.sleep.time", 5000);
}