Java ClusterState.getCollection方法代码示例

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void waitTillRecovered() throws Exception {
  for (int i = 0; i < 30; i++) {
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.updateClusterState(true);
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName())
          || !replica.get(ZkStateReader.STATE_PROP).equals(
          ZkStateReader.ACTIVE)) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see recovered node");
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void waitTillAllNodesActive() throws Exception {
  for (int i = 0; i < 60; i++) { 
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.updateClusterState(true);
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName())
          || !replica.get(ZkStateReader.STATE_PROP).equals(
              ZkStateReader.ACTIVE)) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see all nodes active");
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void waitTillRecovered() throws Exception {
  for (int i = 0; i < 30; i++) { 
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.updateClusterState(true);
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName())
          || !replica.get(ZkStateReader.STATE_PROP).equals(
              ZkStateReader.ACTIVE)) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see recovered node");
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params, NamedList results, String stateMatcher) {
  log.info("Executing Collection Cmd : " + params);
  String collectionName = message.getStr("name");
  
  DocCollection coll = clusterState.getCollection(collectionName);
  
  for (Map.Entry<String,Slice> entry : coll.getSlicesMap().entrySet()) {
    Slice slice = entry.getValue();
    sliceCmd(clusterState, params, stateMatcher, slice);
  }
  
  ShardResponse srsp;
  do {
    srsp = shardHandler.takeCompletedOrError();
    if (srsp != null) {
      processResponse(results, srsp);
    }
  } while (srsp != null);

}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
@Override
public List<? extends BoundedSource<SolrDocument>> split(
    long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
  ConnectionConfiguration connectionConfig = spec.getConnectionConfiguration();
  List<BoundedSolrSource> sources = new ArrayList<>();
  try (AuthorizedSolrClient<CloudSolrClient> client = connectionConfig.createClient()) {
    String collection = spec.getCollection();
    final ClusterState clusterState = AuthorizedSolrClient.getClusterState(client);
    DocCollection docCollection = clusterState.getCollection(collection);
    for (Slice slice : docCollection.getSlices()) {
      ArrayList<Replica> replicas = new ArrayList<>(slice.getReplicas());
      Collections.shuffle(replicas);
      // Load balancing by randomly picking an active replica
      Replica randomActiveReplica = null;
      for (Replica replica : replicas) {
        // We need to check both state of the replica and live nodes
        // to make sure that the replica is alive
        if (replica.getState() == Replica.State.ACTIVE
            && clusterState.getLiveNodes().contains(replica.getNodeName())) {
          randomActiveReplica = replica;
          break;
        }
      }
      // TODO in case of this replica goes inactive while the pipeline runs.
      // We should pick another active replica of this shard.
      checkState(
          randomActiveReplica != null,
          "Can not found an active replica for slice %s",
          slice.getName());
      sources.add(new BoundedSolrSource(spec, randomActiveReplica));
    }
  }
  return sources;
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
protected List<String> buildShardList(CloudSolrClient cloudSolrServer, final String collection) {
  ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection docCollection = clusterState.getCollection(collection);
  Collection<Slice> slices = docCollection.getSlices();

  if (slices == null) {
    throw new IllegalArgumentException("Collection " + collection + " not found!");
  }

  Set<String> liveNodes = clusterState.getLiveNodes();
  Random random = new Random();
  List<String> shards = new ArrayList<String>();
  for (Slice slice : slices) {
    List<String> replicas = new ArrayList<String>();
    for (Replica r : slice.getReplicas()) {
      ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
      if (liveNodes.contains(replicaCoreProps.getNodeName())) {
        replicas.add(replicaCoreProps.getCoreUrl());
      }
    }
    int numReplicas = replicas.size();
    if (numReplicas == 0) {
      throw new IllegalStateException(
          "Shard " + slice.getName() + " does not have any active replicas!");
    }

    String replicaUrl = (numReplicas == 1) ?
        replicas.get(0) :
        replicas.get(random.nextInt(replicas.size()));
    shards.add(replicaUrl);
  }

  return shards;
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private ClusterState removeShard(final ClusterState clusterState, ZkNodeProps message) {
  final String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP);
  final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return clusterState;

  log.info("Removing collection: " + collection + " shard: " + sliceId + " from clusterstate");

  DocCollection coll = clusterState.getCollection(collection);

  Map<String, Slice> newSlices = new LinkedHashMap<>(coll.getSlicesMap());
  newSlices.remove(sliceId);

  DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter());
  return newState(clusterState, singletonMap(collection,newCollection));
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void collectionCmd(ClusterState clusterState, ZkNodeProps message, ModifiableSolrParams params, NamedList results, String stateMatcher) {
  log.info("Executing Collection Cmd : " + params);
  String collectionName = message.getStr("name");
  ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
  
  DocCollection coll = clusterState.getCollection(collectionName);
  
  for (Map.Entry<String,Slice> entry : coll.getSlicesMap().entrySet()) {
    Slice slice = entry.getValue();
    sliceCmd(clusterState, params, stateMatcher, slice, shardHandler);
  }

  processResponses(results, shardHandler);

}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private ClusterState removeShard(final ClusterState clusterState, ZkNodeProps message) {
  final String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP);
  final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return clusterState;

  log.info("Removing collection: " + collection + " shard: " + sliceId + " from clusterstate");

  DocCollection coll = clusterState.getCollection(collection);

  Map<String, Slice> newSlices = new LinkedHashMap<String, Slice>(coll.getSlicesMap());
  newSlices.remove(sliceId);

  DocCollection newCollection = new DocCollection(coll.getName(), newSlices, coll.getProperties(), coll.getRouter());
  return newState(clusterState, singletonMap(collection,newCollection));
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void migrate(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
  String sourceCollectionName = message.getStr("collection");
  String splitKey = message.getStr("split.key");
  String targetCollectionName = message.getStr("target.collection");
  int timeout = message.getInt("forward.timeout", 10 * 60) * 1000;

  DocCollection sourceCollection = clusterState.getCollection(sourceCollectionName);
  if (sourceCollection == null) {
    throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown source collection: " + sourceCollectionName);
  }
  DocCollection targetCollection = clusterState.getCollection(targetCollectionName);
  if (targetCollection == null) {
    throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown target collection: " + sourceCollectionName);
  }
  if (!(sourceCollection.getRouter() instanceof CompositeIdRouter))  {
    throw new SolrException(ErrorCode.BAD_REQUEST, "Source collection must use a compositeId router");
  }
  if (!(targetCollection.getRouter() instanceof CompositeIdRouter))  {
    throw new SolrException(ErrorCode.BAD_REQUEST, "Target collection must use a compositeId router");
  }
  CompositeIdRouter sourceRouter = (CompositeIdRouter) sourceCollection.getRouter();
  CompositeIdRouter targetRouter = (CompositeIdRouter) targetCollection.getRouter();
  Collection<Slice> sourceSlices = sourceRouter.getSearchSlicesSingle(splitKey, null, sourceCollection);
  if (sourceSlices.isEmpty()) {
    throw new SolrException(ErrorCode.BAD_REQUEST,
        "No active slices available in source collection: " + sourceCollection + "for given split.key: " + splitKey);
  }
  Collection<Slice> targetSlices = targetRouter.getSearchSlicesSingle(splitKey, null, targetCollection);
  if (targetSlices.isEmpty()) {
    throw new SolrException(ErrorCode.BAD_REQUEST,
        "No active slices available in target collection: " + targetCollection + "for given split.key: " + splitKey);
  }

  for (Slice sourceSlice : sourceSlices) {
    for (Slice targetSlice : targetSlices) {
      log.info("Migrating source shard: {} to target shard: {} for split.key = " + splitKey, sourceSlice, targetSlice);
      migrateKey(clusterState, sourceCollection, sourceSlice, targetCollection, targetSlice, splitKey, timeout, results);
    }
  }
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
protected static void ensureAllReplicasAreActive(String testCollectionName, int shards, int rf,
    int maxWaitSecs) throws Exception {
  long startMs = System.currentTimeMillis();

  ZkStateReader zkr = cloudSolrClient.getZkStateReader();
  zkr.forceUpdateCollection(testCollectionName); // force the state to be fresh

  ClusterState cs = zkr.getClusterState();
  DocCollection docCollection = cs.getCollection(testCollectionName);
  Collection<Slice> slices = docCollection.getActiveSlices();
  assertTrue(slices.size() == shards);
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  Replica leader = null;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    // refresh state every 2 secs
    if (waitMs % 2000 == 0) {
      log.info("Updating ClusterState");
      cloudSolrClient.getZkStateReader().forceUpdateCollection(testCollectionName);
    }

    cs = cloudSolrClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    allReplicasUp = true; // assume true
    for (Slice shard : docCollection.getActiveSlices()) {
      String shardId = shard.getName();
      assertNotNull("No Slice for " + shardId, shard);
      Collection<Replica> replicas = shard.getReplicas();
      assertTrue(replicas.size() == rf);
      leader = shard.getLeader();
      assertNotNull(leader);
      log.info("Found " + replicas.size() + " replicas and leader on " +
          leader.getNodeName() + " for " + shardId + " in " + testCollectionName);

      // ensure all replicas are "active"
      for (Replica replica : replicas) {
        Replica.State replicaState = replica.getState();
        if (!Replica.State.ACTIVE.equals(replicaState)) {
          log.info("Replica " + replica.getName() + " for shard " + shardId + " is currently "
              + replicaState);
          allReplicasUp = false;
        }
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(500L);
      } catch (Exception ignoreMe) {
      }
      waitMs += 500L;
    }
  } // end while

  if (!allReplicasUp) {
    fail("Didn't see all replicas for " + testCollectionName +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(
        testCollectionName));
  }
  long diffMs = (System.currentTimeMillis() - startMs);
  log.info("Took " + diffMs + " ms to see all replicas become active for " + testCollectionName);
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void addSlices(Map<String,Slice> target, ClusterState state, SolrParams params, String collectionName, String shardKeys, boolean multiCollection) {
  DocCollection coll = state.getCollection(collectionName);
  Collection<Slice> slices = coll.getRouter().getSearchSlices(shardKeys, params , coll);
  ClientUtils.addSlices(target, collectionName, slices, multiCollection);
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void doWork() {
  
  // TODO: extract to configurable strategy class ??
  ClusterState clusterState = zkStateReader.getClusterState();
  if (clusterState != null) {
    if (lastClusterStateVersion == clusterState.getZkClusterStateVersion() && baseUrlForBadNodes.size() == 0) {
      // nothing has changed, no work to do
      return;
    }
    
    lastClusterStateVersion = clusterState.getZkClusterStateVersion();
    Set<String> collections = clusterState.getCollections();
    for (final String collection : collections) {
      DocCollection docCollection = clusterState.getCollection(collection);
      if (!docCollection.getAutoAddReplicas()) {
        continue;
      }
      if (docCollection.getReplicationFactor() == null) {
        log.debug("Skipping collection because it has no defined replicationFactor, name={}", docCollection.getName());
        continue;
      }
      log.debug("Found collection, name={} replicationFactor=", collection, docCollection.getReplicationFactor());
      
      Collection<Slice> slices = docCollection.getSlices();
      for (Slice slice : slices) {
        if (slice.getState().equals(Slice.ACTIVE)) {
          
          final Collection<DownReplica> downReplicas = new ArrayList<DownReplica>();
          
          int goodReplicas = findDownReplicasInSlice(clusterState, docCollection, slice, downReplicas);
          
          log.debug("replicationFactor={} goodReplicaCount={}", docCollection.getReplicationFactor(), goodReplicas);
          
          if (downReplicas.size() > 0 && goodReplicas < docCollection.getReplicationFactor()) {
            // badReplicaMap.put(collection, badReplicas);
            processBadReplicas(collection, downReplicas);
          } else if (goodReplicas > docCollection.getReplicationFactor()) {
            log.debug("There are too many replicas");
          }
        }
      }
    }
   
  }
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
  checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP,REPLICA_PROP);
  String collectionName = message.getStr(COLLECTION_PROP);
  String shard = message.getStr(SHARD_ID_PROP);
  String replicaName = message.getStr(REPLICA_PROP);
  DocCollection coll = clusterState.getCollection(collectionName);
  Slice slice = coll.getSlice(shard);
  ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
  if(slice==null){
    throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid shard name : "+shard+" in collection : "+ collectionName);
  }
  Replica replica = slice.getReplica(replicaName);
  if(replica == null){
    ArrayList<String> l = new ArrayList<>();
    for (Replica r : slice.getReplicas()) l.add(r.getName());
    throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid replica : " + replicaName + " in shard/collection : "
        + shard + "/"+ collectionName + " available replicas are "+ StrUtils.join(l,','));
  }

  String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
  String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
  
  // assume the core exists and try to unload it
  Map m = ZkNodeProps.makeMap("qt", adminPath, CoreAdminParams.ACTION,
      CoreAdminAction.UNLOAD.toString(), CoreAdminParams.CORE, core,
      CoreAdminParams.DELETE_INSTANCE_DIR, "true",
      CoreAdminParams.DELETE_DATA_DIR, "true");
  
  ShardRequest sreq = new ShardRequest();
  sreq.purpose = 1;
  sreq.shards = new String[] {baseUrl};
  sreq.actualShards = sreq.shards;
  sreq.params = new ModifiableSolrParams(new MapSolrParams(m));
  try {
    shardHandler.submit(sreq, baseUrl, sreq.params);
  } catch (Exception e) {
    log.warn("Exception trying to unload core " + sreq, e);
  }
  
  collectShardResponses(!Slice.ACTIVE.equals(replica.getStr(Slice.STATE)) ? new NamedList() : results,
      false, null, shardHandler);
  
  if (waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return;//check if the core unload removed the corenode zk enry
  deleteCoreNode(collectionName, replicaName, replica, core); // try and ensure core info is removed from clusterstate
  if(waitForCoreNodeGone(collectionName, shard, replicaName, 30000)) return;

  throw new SolrException(ErrorCode.SERVER_ERROR, "Could not  remove replica : " + collectionName + "/" + shard+"/" + replicaName);
}
 

import org.apache.solr.common.cloud.ClusterState; //导入方法依赖的package包/类
private boolean createShard(ClusterState clusterState, ZkNodeProps message, NamedList results)
    throws KeeperException, InterruptedException {
  log.info("Create shard invoked: {}", message);
  String collectionName = message.getStr(COLLECTION_PROP);
  String shard = message.getStr(SHARD_ID_PROP);
  if(collectionName == null || shard ==null)
    throw new SolrException(ErrorCode.BAD_REQUEST, "'collection' and 'shard' are required parameters" );
  int numSlices = 1;

  ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
  DocCollection collection = clusterState.getCollection(collectionName);
  int maxShardsPerNode = collection.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1);
  int repFactor = message.getInt(ZkStateReader.REPLICATION_FACTOR, collection.getInt(ZkStateReader.REPLICATION_FACTOR, 1));
  String createNodeSetStr = message.getStr(CREATE_NODE_SET);

  ArrayList<Node> sortedNodeList = getNodesForNewShard(clusterState, collectionName, numSlices, maxShardsPerNode, repFactor, createNodeSetStr);

  Overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(message));
  // wait for a while until we see the shard
  long waitUntil = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);;
  boolean created = false;
  while (System.nanoTime() < waitUntil) {
    Thread.sleep(100);
    created = zkStateReader.getClusterState().getCollection(collectionName).getSlice(shard) != null;
    if (created) break;
  }
  if (!created)
    throw new SolrException(ErrorCode.SERVER_ERROR, "Could not fully create shard: " + message.getStr("name"));


  String configName = message.getStr(COLL_CONF);
  String sliceName = shard;
  for (int j = 1; j <= repFactor; j++) {
    String nodeName = sortedNodeList.get(((j - 1)) % sortedNodeList.size()).nodeName;
    String shardName = collectionName + "_" + sliceName + "_replica" + j;
    log.info("Creating shard " + shardName + " as part of slice "
        + sliceName + " of collection " + collectionName + " on "
        + nodeName);

    // Need to create new params for each request
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(CoreAdminParams.ACTION, CoreAdminAction.CREATE.toString());

    params.set(CoreAdminParams.NAME, shardName);
    params.set(COLL_CONF, configName);
    params.set(CoreAdminParams.COLLECTION, collectionName);
    params.set(CoreAdminParams.SHARD, sliceName);
    params.set(ZkStateReader.NUM_SHARDS_PROP, numSlices);
    addPropertyParams(message, params);

    ShardRequest sreq = new ShardRequest();
    params.set("qt", adminPath);
    sreq.purpose = 1;
    String replica = zkStateReader.getBaseUrlForNodeName(nodeName);
    sreq.shards = new String[]{replica};
    sreq.actualShards = sreq.shards;
    sreq.params = params;

    shardHandler.submit(sreq, replica, sreq.params);

  }

  processResponses(results, shardHandler);

  log.info("Finished create command on all shards for collection: "
      + collectionName);

  return true;
}