Java DocCollection.getSlices方法代码示例

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private List<Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
  Collection<Slice> allSlices = coll.getSlices();
  List<Node> nodes = null;
  for (Slice aslice : allSlices) {
    if (Slice.CONSTRUCTION.equals(aslice.getState()) || Slice.RECOVERY.equals(aslice.getState()))  {
      DocRouter.Range myRange = coll.getSlice(shardId).getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
      if (isSubset &&
          (docId == null // in case of deletes
          || (docId != null && coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll)))) {
        Replica sliceLeader = aslice.getLeader();
        // slice leader can be null because node/shard is created zk before leader election
        if (sliceLeader != null && zkController.getClusterState().liveNodesContain(sliceLeader.getNodeName()))  {
          if (nodes == null) nodes = new ArrayList<>();
          ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(sliceLeader);
          nodes.add(new StdNode(nodeProps, coll.getName(), shardId));
        }
      }
    }
  }
  return nodes;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private boolean checkIfCoreNodeNameAlreadyExists(CoreDescriptor dcore) {
  ZkStateReader zkStateReader = coreContainer.getZkController()
      .getZkStateReader();
  DocCollection collection = zkStateReader.getClusterState().getCollectionOrNull(dcore.getCollectionName());
  if (collection != null) {
    Collection<Slice> slices = collection.getSlices();
    
    for (Slice slice : slices) {
      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        if (replica.getName().equals(
            dcore.getCloudDescriptor().getCoreNodeName())) {
          return true;
        }
      }
    }
  }
  return false;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
/** Returns all replicas per node. */
private Map<String, List<ReplicaInfo>> getNodeReplicas() {
    final ClusterState clusterState = getClusterState();
    final Map<String, List<ReplicaInfo>> result = Maps.newHashMap();
    for (final DocCollection collection : clusterState.getCollectionsMap().values()) {
        for (final Slice slice : collection.getSlices()) {
            for (final Replica replica : slice.getReplicas()) {
                List<ReplicaInfo> nodeReplicas = result.get(replica.getNodeName());
                if (nodeReplicas == null) {
                    nodeReplicas = Lists.newArrayList();
                    result.put(replica.getNodeName(), nodeReplicas);
                }
                nodeReplicas.add(new ReplicaInfo(replica, collection.getName(), slice.getName()));
            }
        }
    }
    return result;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private List<Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
  Collection<Slice> allSlices = coll.getSlices();
  List<Node> nodes = null;
  for (Slice aslice : allSlices) {
    if (Slice.CONSTRUCTION.equals(aslice.getState()) || Slice.RECOVERY.equals(aslice.getState()))  {
      DocRouter.Range myRange = coll.getSlice(shardId).getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
      if (isSubset &&
          (docId == null // in case of deletes
          || (docId != null && coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll)))) {
        Replica sliceLeader = aslice.getLeader();
        // slice leader can be null because node/shard is created zk before leader election
        if (sliceLeader != null && zkController.getClusterState().liveNodesContain(sliceLeader.getNodeName()))  {
          if (nodes == null) nodes = new ArrayList<Node>();
          ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(sliceLeader);
          nodes.add(new StdNode(nodeProps));
        }
      }
    }
  }
  return nodes;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
@Override
public List<? extends BoundedSource<SolrDocument>> split(
    long desiredBundleSizeBytes, PipelineOptions options) throws Exception {
  ConnectionConfiguration connectionConfig = spec.getConnectionConfiguration();
  List<BoundedSolrSource> sources = new ArrayList<>();
  try (AuthorizedSolrClient<CloudSolrClient> client = connectionConfig.createClient()) {
    String collection = spec.getCollection();
    final ClusterState clusterState = AuthorizedSolrClient.getClusterState(client);
    DocCollection docCollection = clusterState.getCollection(collection);
    for (Slice slice : docCollection.getSlices()) {
      ArrayList<Replica> replicas = new ArrayList<>(slice.getReplicas());
      Collections.shuffle(replicas);
      // Load balancing by randomly picking an active replica
      Replica randomActiveReplica = null;
      for (Replica replica : replicas) {
        // We need to check both state of the replica and live nodes
        // to make sure that the replica is alive
        if (replica.getState() == Replica.State.ACTIVE
            && clusterState.getLiveNodes().contains(replica.getNodeName())) {
          randomActiveReplica = replica;
          break;
        }
      }
      // TODO in case of this replica goes inactive while the pipeline runs.
      // We should pick another active replica of this shard.
      checkState(
          randomActiveReplica != null,
          "Can not found an active replica for slice %s",
          slice.getName());
      sources.add(new BoundedSolrSource(spec, randomActiveReplica));
    }
  }
  return sources;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
protected List<String> buildShardList(CloudSolrClient cloudSolrServer, final String collection) {
  ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection docCollection = clusterState.getCollection(collection);
  Collection<Slice> slices = docCollection.getSlices();

  if (slices == null) {
    throw new IllegalArgumentException("Collection " + collection + " not found!");
  }

  Set<String> liveNodes = clusterState.getLiveNodes();
  Random random = new Random();
  List<String> shards = new ArrayList<String>();
  for (Slice slice : slices) {
    List<String> replicas = new ArrayList<String>();
    for (Replica r : slice.getReplicas()) {
      ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
      if (liveNodes.contains(replicaCoreProps.getNodeName())) {
        replicas.add(replicaCoreProps.getCoreUrl());
      }
    }
    int numReplicas = replicas.size();
    if (numReplicas == 0) {
      throw new IllegalStateException(
          "Shard " + slice.getName() + " does not have any active replicas!");
    }

    String replicaUrl = (numReplicas == 1) ?
        replicas.get(0) :
        replicas.get(random.nextInt(replicas.size()));
    shards.add(replicaUrl);
  }

  return shards;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
protected int getTotalReplicas(String collection) {
  ZkStateReader zkStateReader = cloudClient.getZkStateReader();
  DocCollection coll = zkStateReader.getClusterState().getCollectionOrNull(collection);
  if (coll == null) return 0;  // support for when collection hasn't been created yet
  int cnt = 0;
  for (Slice slices : coll.getSlices()) {
    cnt += slices.getReplicas().size();
  }
  return cnt;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private Map<String, Replica> waitToSeeReplicasInState(String collectionName, Collection<String> coreNames) throws InterruptedException {
  Map<String, Replica> result = new HashMap<>();
  long endTime = System.nanoTime() + TimeUnit.NANOSECONDS.convert(30, TimeUnit.SECONDS);
  while (true) {
    DocCollection coll = zkStateReader.getClusterState().getCollection(
        collectionName);
    for (String coreName : coreNames) {
      if (result.containsKey(coreName)) continue;
      for (Slice slice : coll.getSlices()) {
        for (Replica replica : slice.getReplicas()) {
          if (coreName.equals(replica.getStr(ZkStateReader.CORE_NAME_PROP))) {
            result.put(coreName, replica);
            break;
          }
        }
      }
    }
    
    if (result.size() == coreNames.size()) {
      return result;
    }
    if (System.nanoTime() > endTime) {
      throw new SolrException(ErrorCode.SERVER_ERROR, "Timed out waiting to see all replicas in cluster state.");
    }
    
    Thread.sleep(100);
  }
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private List<Replica> getReplicas() {
  List<Replica> replicas = new ArrayList<Replica>();
  
  DocCollection collection = this.cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION);
  for(Slice slice : collection.getSlices()) {
    replicas.addAll(slice.getReplicas());
  }
  return replicas;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private void deleteLiveReplicaTest() throws Exception {
  String collectionName = "delLiveColl";
  CloudSolrServer client = createCloudClient(null);
  try {
    createCollection(collectionName, client);
    
    waitForRecoveriesToFinish(collectionName, false);
    
    DocCollection testcoll = getCommonCloudSolrServer().getZkStateReader()
        .getClusterState().getCollection(collectionName);
    
    Slice shard1 = null;
    Replica replica1 = null;
    for (Slice slice : testcoll.getSlices()) {
      if ("active".equals(slice.getStr("state"))) {
        shard1 = slice;
        for (Replica replica : shard1.getReplicas())
          if ("active".equals(replica.getStr("state"))) replica1 = replica;
      }
    }

    if (replica1 == null) fail("no active replicas found");

    HttpSolrServer replica1Server = new HttpSolrServer(replica1.getStr("base_url"));
    String dataDir = null;
    try {
      CoreAdminResponse status = CoreAdminRequest.getStatus(replica1.getStr("core"), replica1Server);
      NamedList<Object> coreStatus = status.getCoreStatus(replica1.getStr("core"));
      dataDir = (String) coreStatus.get("dataDir");
    } finally {
      replica1Server.shutdown();
    }

    removeAndWaitForReplicaGone(collectionName, client, replica1,
        shard1.getName());
    assertFalse("dataDir for " + replica1.getName() + " should have been deleted by deleteReplica API", new File(dataDir).exists());
  } finally {
    client.shutdown();
  }
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
/** Returns all the replicas (of all shards and collections) that exist on the given node. */
public List<Replica> getAllNodeReplicas(String nodeName) {
    final List<Replica> replicas = Lists.newArrayList();
    final ClusterState clusterState = getClusterState();
    for (final DocCollection collection : clusterState.getCollectionsMap().values()) {
        for (final Slice slice : collection.getSlices()) {
            for (final Replica replica : slice.getReplicas()) {
                if (replica.getNodeName().equals(nodeName)) {
                    replicas.add(replica);
                }
            }
        }
    }
    return replicas;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
protected int getTotalReplicas(String collection) {
  ZkStateReader zkStateReader = cloudClient.getZkStateReader();
  DocCollection coll = zkStateReader.getClusterState().getCollectionStates().get(collection);
  if (coll == null) return 0;  // support for when collection hasn't been created yet
  int cnt = 0;
  for (Slice slices : coll.getSlices()) {
    cnt += slices.getReplicas().size();
  }
  return cnt;
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private void doWork() {
  
  // TODO: extract to configurable strategy class ??
  ClusterState clusterState = zkStateReader.getClusterState();
  if (clusterState != null) {
    if (lastClusterStateVersion == clusterState.getZkClusterStateVersion() && baseUrlForBadNodes.size() == 0) {
      // nothing has changed, no work to do
      return;
    }
    
    lastClusterStateVersion = clusterState.getZkClusterStateVersion();
    Set<String> collections = clusterState.getCollections();
    for (final String collection : collections) {
      DocCollection docCollection = clusterState.getCollection(collection);
      if (!docCollection.getAutoAddReplicas()) {
        continue;
      }
      if (docCollection.getReplicationFactor() == null) {
        log.debug("Skipping collection because it has no defined replicationFactor, name={}", docCollection.getName());
        continue;
      }
      log.debug("Found collection, name={} replicationFactor=", collection, docCollection.getReplicationFactor());
      
      Collection<Slice> slices = docCollection.getSlices();
      for (Slice slice : slices) {
        if (slice.getState().equals(Slice.ACTIVE)) {
          
          final Collection<DownReplica> downReplicas = new ArrayList<DownReplica>();
          
          int goodReplicas = findDownReplicasInSlice(clusterState, docCollection, slice, downReplicas);
          
          log.debug("replicationFactor={} goodReplicaCount={}", docCollection.getReplicationFactor(), goodReplicas);
          
          if (downReplicas.size() > 0 && goodReplicas < docCollection.getReplicationFactor()) {
            // badReplicaMap.put(collection, badReplicas);
            processBadReplicas(collection, downReplicas);
          } else if (goodReplicas > docCollection.getReplicationFactor()) {
            log.debug("There are too many replicas");
          }
        }
      }
    }
   
  }
}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private void testNodesUsedByCreate() throws Exception {
  // we can use this client because we just want base url
  final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
  
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set("action", CollectionAction.CREATE.toString());

  params.set("numShards", 2);
  params.set(REPLICATION_FACTOR, 2);
  String collectionName = "nodes_used_collection";

  params.set("name", collectionName);
  
  if (secondConfigSet) {
    params.set("collection.configName", "conf1");
  }
  
  QueryRequest request = new QueryRequest(params);
  request.setPath("/admin/collections");
  createNewSolrServer("", baseUrl).request(request);
  
  List<Integer> numShardsNumReplicaList = new ArrayList<>();
  numShardsNumReplicaList.add(2);
  numShardsNumReplicaList.add(2);
  checkForCollection("nodes_used_collection", numShardsNumReplicaList , null);

  List<String> createNodeList = new ArrayList<>();

  Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState()
      .getLiveNodes();
  
  for (String node : liveNodes) {
    createNodeList.add(node);
  }

  DocCollection col = cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
  Collection<Slice> slices = col.getSlices();
  for (Slice slice : slices) {
    Collection<Replica> replicas = slice.getReplicas();
    for (Replica replica : replicas) {
      createNodeList.remove(replica.getNodeName());
    }
  }
  assertEquals(createNodeList.toString(), 1, createNodeList.size());

}
 

import org.apache.solr.common.cloud.DocCollection; //导入方法依赖的package包/类
private void testNodesUsedByCreate() throws Exception {
  // we can use this client because we just want base url
  final String baseUrl = getBaseUrl((HttpSolrServer) clients.get(0));
  
  ModifiableSolrParams params = new ModifiableSolrParams();
  params.set("action", CollectionAction.CREATE.toString());

  params.set("numShards", 2);
  params.set(OverseerCollectionProcessor.REPLICATION_FACTOR, 2);
  String collectionName = "nodes_used_collection";

  params.set("name", collectionName);
  QueryRequest request = new QueryRequest(params);
  request.setPath("/admin/collections");
  createNewSolrServer("", baseUrl).request(request);
  
  List<Integer> numShardsNumReplicaList = new ArrayList<Integer>();
  numShardsNumReplicaList.add(2);
  numShardsNumReplicaList.add(2);
  checkForCollection("nodes_used_collection", numShardsNumReplicaList , null);
  
  List<String> createNodeList = new ArrayList<String>();

  Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState()
      .getLiveNodes();
  
  for (String node : liveNodes) {
    createNodeList.add(node);
  }
  
  DocCollection col = cloudClient.getZkStateReader().getClusterState().getCollection("nodes_used_collection");
  Collection<Slice> slices = col.getSlices();
  for (Slice slice : slices) {
    Collection<Replica> replicas = slice.getReplicas();
    for (Replica replica : replicas) {
      createNodeList.remove(replica.getNodeName());
    }
  }
  assertEquals(createNodeList.toString(), 1, createNodeList.size());

}