Java HelixAdmin.getResourceIdealState方法代码示例

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * From IdealStates.
 *
 * @return InstanceToNumTopicPartitionMap
 */
public static Map<String, Set<TopicPartition>> getInstanceToTopicPartitionsMap(
    HelixManager helixManager) {
  Map<String, Set<TopicPartition>> instanceToNumTopicPartitionMap =
      new HashMap<String, Set<TopicPartition>>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    for (String partition : is.getPartitionSet()) {
      TopicPartition tpi = new TopicPartition(topic, Integer.parseInt(partition));
      for (String instance : is.getInstanceSet(partition)) {
        if (!instanceToNumTopicPartitionMap.containsKey(instance)) {
          instanceToNumTopicPartitionMap.put(instance, new HashSet<TopicPartition>());
        }
        instanceToNumTopicPartitionMap.get(instance).add(tpi);
      }
    }
  }
  return instanceToNumTopicPartitionMap;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) {
  Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    int numPartitions = is.getNumPartitions();
    for (int partition = 0; partition < numPartitions; ++partition) {
      if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) {
        TopicPartition tpi = new TopicPartition(topic, partition);
        unassignedPartitions.add(tpi);
      }
    }
  }
  return unassignedPartitions;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * From IdealStates.
 * @param helixManager
 * @return InstanceToNumTopicPartitionMap
 */
public static Map<String, Set<TopicPartition>> getInstanceToTopicPartitionsMap(
    HelixManager helixManager) {
  Map<String, Set<TopicPartition>> instanceToNumTopicPartitionMap =
      new HashMap<String, Set<TopicPartition>>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    for (String partition : is.getPartitionSet()) {
      TopicPartition tpi = new TopicPartition(topic, Integer.parseInt(partition));
      for (String instance : is.getInstanceSet(partition)) {
        if (!instanceToNumTopicPartitionMap.containsKey(instance)) {
          instanceToNumTopicPartitionMap.put(instance, new HashSet<TopicPartition>());
        }
        instanceToNumTopicPartitionMap.get(instance).add(tpi);
      }
    }
  }
  return instanceToNumTopicPartitionMap;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * Remove a segment is also required to recompute the ideal state.
 *
 * @param tableName
 * @param segmentId
 * @param helixAdmin
 * @param helixClusterName
 * @return
 */
public synchronized static IdealState dropSegmentFromIdealStateFor(String tableName, String segmentId,
    HelixAdmin helixAdmin, String helixClusterName) {

  final IdealState currentIdealState = helixAdmin.getResourceIdealState(helixClusterName, tableName);
  final Set<String> currentInstanceSet = currentIdealState.getInstanceSet(segmentId);
  if (!currentInstanceSet.isEmpty() && currentIdealState.getPartitionSet().contains(segmentId)) {
    for (String instanceName : currentIdealState.getInstanceSet(segmentId)) {
      currentIdealState.setPartitionState(segmentId, instanceName, "DROPPED");
    }
  } else {
    throw new RuntimeException("Cannot found segmentId - " + segmentId + " in table - " + tableName);
  }
  return currentIdealState;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * Remove a segment is also required to recompute the ideal state.
 *
 * @param tableName
 * @param segmentId
 * @param helixAdmin
 * @param helixClusterName
 * @return
 */
public synchronized static IdealState removeSegmentFromIdealStateFor(String tableName, String segmentId,
    HelixAdmin helixAdmin, String helixClusterName) {

  final IdealState currentIdealState = helixAdmin.getResourceIdealState(helixClusterName, tableName);
  if (currentIdealState != null && currentIdealState.getPartitionSet() != null
      && currentIdealState.getPartitionSet().contains(segmentId)) {
    currentIdealState.getPartitionSet().remove(segmentId);
  } else {
    throw new RuntimeException("Cannot found segmentId - " + segmentId + " in table - " + tableName);
  }
  return currentIdealState;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 *
 * @param brokerResourceName
 * @param helixAdmin
 * @param helixClusterName
 * @return
 */
public static IdealState removeBrokerResourceFromIdealStateFor(String brokerResourceName, HelixAdmin helixAdmin,
    String helixClusterName) {
  final IdealState currentIdealState =
      helixAdmin.getResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
  final Set<String> currentInstanceSet = currentIdealState.getInstanceSet(brokerResourceName);
  if (!currentInstanceSet.isEmpty() && currentIdealState.getPartitionSet().contains(brokerResourceName)) {
    currentIdealState.getPartitionSet().remove(brokerResourceName);
  } else {
    throw new RuntimeException("Cannot found broker resource - " + brokerResourceName + " in broker resource ");
  }
  return currentIdealState;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Test
public void testEnableDisablePartitions() throws InterruptedException {
  HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
  admin.enablePartition(false, CLUSTER_NAME, (PARTICIPANT_PREFIX + "_" + _startPort),
      WorkflowGenerator.DEFAULT_TGT_DB, Arrays.asList(new String[] { "TestDB_0", "TestDB_2" }));

  IdealState idealState =
      admin.getResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  List<String> preferenceList =
      Arrays.asList(new String[] { "localhost_12919", "localhost_12918" });
  for (String partitionName : idealState.getPartitionSet()) {
    idealState.setPreferenceList(partitionName, preferenceList);
  }
  idealState.setRebalanceMode(IdealState.RebalanceMode.SEMI_AUTO);
  admin.setResourceIdealState(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, idealState);

  String workflowName = TestHelper.getTestMethodName();
  Workflow.Builder builder = new Workflow.Builder(workflowName);
  JobConfig.Builder jobBuilder =
      new JobConfig.Builder().setWorkflow(workflowName).setCommand(MockTask.TASK_COMMAND)
          .setTargetResource(WorkflowGenerator.DEFAULT_TGT_DB)
          .setTargetPartitionStates(Collections.singleton("SLAVE"));
  builder.addJob("JOB", jobBuilder);
  _driver.start(builder.build());
  Thread.sleep(2000L);
  JobContext jobContext =
      _driver.getJobContext(TaskUtil.getNamespacedJobName(workflowName, "JOB"));
  Assert.assertEquals(jobContext.getPartitionState(0), null);
  Assert.assertEquals(jobContext.getPartitionState(1), TaskPartitionState.COMPLETED);
  Assert.assertEquals(jobContext.getPartitionState(2), null);
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
private Set<String> getPartitionsAssignedtoInstance(String cluster, String dbName, String instance) {
  HelixAdmin admin = _setupTool.getClusterManagementTool();
  Set<String> partitionSet = new HashSet<String>();
  IdealState is = admin.getResourceIdealState(cluster, dbName);
  for (String partition : is.getRecord().getListFields().keySet()) {
    List<String> assignments = is.getRecord().getListField(partition);
    for (String ins : assignments) {
      if (ins.equals(instance)) {
        partitionSet.add(partition);
      }
    }
  }

  return partitionSet;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@GET
@Path("{resourceName}")
public Response getResource(@PathParam("clusterId") String clusterId,
    @PathParam("resourceName") String resourceName) throws IOException {
  ConfigAccessor accessor = getConfigAccessor();
  HelixAdmin admin = getHelixAdmin();

  ResourceConfig resourceConfig = accessor.getResourceConfig(clusterId, resourceName);
  IdealState idealState = admin.getResourceIdealState(clusterId, resourceName);
  ExternalView externalView = admin.getResourceExternalView(clusterId, resourceName);

  Map<String, ZNRecord> resourceMap = new HashMap<>();
  if (idealState != null) {
    resourceMap.put(ResourceProperties.idealState.name(), idealState.getRecord());
  } else {
    return notFound();
  }

  resourceMap.put(ResourceProperties.resourceConfig.name(), null);
  resourceMap.put(ResourceProperties.externalView.name(), null);

  if (resourceConfig != null) {
    resourceMap.put(ResourceProperties.resourceConfig.name(), resourceConfig.getRecord());
  }

  if (externalView != null) {
    resourceMap.put(ResourceProperties.externalView.name(), externalView.getRecord());
  }

  return JSONRepresentation(resourceMap);
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@GET
@Path("{resourceName}/idealState")
public Response getResourceIdealState(@PathParam("clusterId") String clusterId,
    @PathParam("resourceName") String resourceName) {
  HelixAdmin admin = getHelixAdmin();
  IdealState idealState = admin.getResourceIdealState(clusterId, resourceName);
  if (idealState != null) {
    return JSONRepresentation(idealState.getRecord());
  }

  return notFound();
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static IdealState getBrokerIdealStates(HelixAdmin admin, String clusterName) {
  return admin.getResourceIdealState(clusterName, BROKER_RESOURCE);
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * Posts new job to cluster
 */
private void scheduleSingleJob(String jobResource, JobConfig jobConfig) {
  HelixAdmin admin = _manager.getClusterManagmentTool();

  IdealState jobIS = admin.getResourceIdealState(_manager.getClusterName(), jobResource);
  if (jobIS != null) {
    LOG.info("Job " + jobResource + " idealstate already exists!");
    return;
  }

  // Set up job resource based on partitions from target resource
  TaskUtil.createUserContent(_manager.getHelixPropertyStore(), jobResource,
      new ZNRecord(TaskUtil.USER_CONTENT_NODE));
  int numIndependentTasks = jobConfig.getTaskConfigMap().size();

  int numPartitions = numIndependentTasks;
  if (numPartitions == 0) {
    IdealState targetIs =
        admin.getResourceIdealState(_manager.getClusterName(), jobConfig.getTargetResource());
    if (targetIs == null) {
      LOG.warn("Target resource does not exist for job " + jobResource);
      // do not need to fail here, the job will be marked as failure immediately when job starts running.
    } else {
      numPartitions = targetIs.getPartitionSet().size();
    }
  }

  admin.addResource(_manager.getClusterName(), jobResource, numPartitions,
      TaskConstants.STATE_MODEL_NAME);

  HelixDataAccessor accessor = _manager.getHelixDataAccessor();

  // Set the job configuration
  PropertyKey.Builder keyBuilder = accessor.keyBuilder();
  HelixProperty resourceConfig = new HelixProperty(jobResource);
  resourceConfig.getRecord().getSimpleFields().putAll(jobConfig.getResourceConfigMap());
  Map<String, TaskConfig> taskConfigMap = jobConfig.getTaskConfigMap();
  if (taskConfigMap != null) {
    for (TaskConfig taskConfig : taskConfigMap.values()) {
      resourceConfig.getRecord().setMapField(taskConfig.getId(), taskConfig.getConfigMap());
    }
  }
  accessor.setProperty(keyBuilder.resourceConfig(jobResource), resourceConfig);

  // Push out new ideal state based on number of target partitions
  IdealStateBuilder builder = new CustomModeISBuilder(jobResource);
  builder.setRebalancerMode(IdealState.RebalanceMode.TASK);
  builder.setNumReplica(1);
  builder.setNumPartitions(numPartitions);
  builder.setStateModel(TaskConstants.STATE_MODEL_NAME);

  if (jobConfig.getInstanceGroupTag() != null) {
    builder.setNodeGroup(jobConfig.getInstanceGroupTag());
  }

  if (jobConfig.isDisableExternalView()) {
    builder.disableExternalView();
  }

  jobIS = builder.build();
  for (int i = 0; i < numPartitions; i++) {
    jobIS.getRecord().setListField(jobResource + "_" + i, new ArrayList<String>());
    jobIS.getRecord().setMapField(jobResource + "_" + i, new HashMap<String, String>());
  }
  jobIS.setRebalancerClassName(JobRebalancer.class.getName());
  admin.setResourceIdealState(_manager.getClusterName(), jobResource, jobIS);
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * Ensure that no assignments happen when there are no tagged nodes, but the resource is tagged
 */
@Test
public void testResourceTaggedFirst() throws Exception {
  final int NUM_PARTICIPANTS = 10;
  final int NUM_PARTITIONS = 4;
  final int NUM_REPLICAS = 2;
  final String RESOURCE_NAME = "TestDB0";
  final String TAG = "ASSIGNABLE";

  String className = TestHelper.getTestClassName();
  String methodName = TestHelper.getTestMethodName();
  String clusterName = className + "_" + methodName;
  System.out.println("START " + clusterName + " at " + new Date(System.currentTimeMillis()));

  // Set up cluster
  TestHelper.setupCluster(clusterName, ZK_ADDR, 12918, // participant port
      "localhost", // participant name prefix
      "TestDB", // resource name prefix
      1, // resources
      NUM_PARTITIONS, // partitions per resource
      NUM_PARTICIPANTS, // number of nodes
      NUM_REPLICAS, // replicas
      "MasterSlave", RebalanceMode.FULL_AUTO, // use FULL_AUTO mode to test node tagging
      true); // do rebalance

  // tag the resource
  HelixAdmin helixAdmin = new ZKHelixAdmin(ZK_ADDR);
  IdealState idealState = helixAdmin.getResourceIdealState(clusterName, RESOURCE_NAME);
  idealState.setInstanceGroupTag(TAG);
  helixAdmin.setResourceIdealState(clusterName, RESOURCE_NAME, idealState);

  // start controller
  ClusterControllerManager controller =
      new ClusterControllerManager(ZK_ADDR, clusterName, "controller");
  controller.syncStart();

  // start participants
  MockParticipantManager[] participants = new MockParticipantManager[NUM_PARTICIPANTS];
  for (int i = 0; i < NUM_PARTICIPANTS; i++) {
    final String instanceName = "localhost_" + (12918 + i);

    participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
    participants[i].syncStart();
  }

  Thread.sleep(1000);
  boolean result =
      ClusterStateVerifier.verifyByZkCallback(new EmptyZkVerifier(clusterName, RESOURCE_NAME));
  Assert.assertTrue(result, "External view and current state must be empty");

  // cleanup
  for (int i = 0; i < NUM_PARTICIPANTS; i++) {
    participants[i].syncStop();
  }
  controller.syncStop();
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Override
public List<String> getAssignedInstances(HelixAdmin helixAdmin, ZkHelixPropertyStore<ZNRecord> propertyStore,
    String helixClusterName, SegmentMetadata segmentMetadata, int numReplicas, String tenantName) {
  String offlineTableName = TableNameBuilder.OFFLINE.tableNameWithType(segmentMetadata.getTableName());
  String serverTenantName = ControllerTenantNameBuilder.getOfflineTenantNameForTenant(tenantName);

  List<String> selectedInstances = new ArrayList<>();
  Map<String, Integer> currentNumSegmentsPerInstanceMap = new HashMap<>();
  List<String> allTaggedInstances =
      HelixHelper.getEnabledInstancesWithTag(helixAdmin, helixClusterName, serverTenantName);

  for (String instance : allTaggedInstances) {
    currentNumSegmentsPerInstanceMap.put(instance, 0);
  }

  // Count number of segments assigned to each instance
  IdealState idealState = helixAdmin.getResourceIdealState(helixClusterName, offlineTableName);
  if (idealState != null) {
    for (String partitionName : idealState.getPartitionSet()) {
      Map<String, String> instanceToStateMap = idealState.getInstanceStateMap(partitionName);
      if (instanceToStateMap != null) {
        for (String instanceName : instanceToStateMap.keySet()) {
          if (currentNumSegmentsPerInstanceMap.containsKey(instanceName)) {
            currentNumSegmentsPerInstanceMap.put(instanceName,
                currentNumSegmentsPerInstanceMap.get(instanceName) + 1);
          }
          // else, ignore. Do not add servers, that are not tagged, to the map
          // By this approach, new segments will not be allotted to the server if tags changed
        }
      }
    }
  }

  // Select up to numReplicas instances with the fewest segments assigned
  PriorityQueue<Number2ObjectPair<String>> priorityQueue =
      new PriorityQueue<>(numReplicas, Pairs.getDescendingnumber2ObjectPairComparator());
  for (String key : currentNumSegmentsPerInstanceMap.keySet()) {
    priorityQueue.add(new Number2ObjectPair<>(currentNumSegmentsPerInstanceMap.get(key), key));
    if (priorityQueue.size() > numReplicas) {
      priorityQueue.poll();
    }
  }

  while (!priorityQueue.isEmpty()) {
    selectedInstances.add(priorityQueue.poll().getB());
  }

  LOGGER.info("Segment assignment result for : " + segmentMetadata.getName() + ", in resource : "
      + segmentMetadata.getTableName() + ", selected instances: " + Arrays.toString(selectedInstances.toArray()));
  return selectedInstances;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
/**
 * Verify that the partition layout information is in sync.
 * @param dc the datacenter whose information is to be verified.
 * @param clusterName the cluster to be verified.
 * @param partitionLayout the {@link PartitionLayout} of the static clustermap.
 */
private void verifyResourcesAndPartitionEquivalencyInDc(Datacenter dc, String clusterName,
    PartitionLayout partitionLayout) {
  String dcName = dc.getName();
  HelixAdmin admin = adminForDc.get(dc.getName());
  Map<String, Set<String>> allPartitionsToInstancesInHelix = new HashMap<>();
  for (String resourceName : admin.getResourcesInCluster(clusterName)) {
    IdealState resourceIS = admin.getResourceIdealState(clusterName, resourceName);
    ensureOrThrow(resourceIS.getStateModelDefRef().equals(LeaderStandbySMD.name),
        "StateModel name mismatch for resource " + resourceName);
    int numReplicasAtResourceLevel = Integer.valueOf(resourceIS.getReplicas());
    Set<String> resourcePartitions = resourceIS.getPartitionSet();
    for (String resourcePartition : resourcePartitions) {
      Set<String> partitionInstanceSet = resourceIS.getInstanceSet(resourcePartition);
      ensureOrThrow(numReplicasAtResourceLevel == partitionInstanceSet.size(),
          "NumReplicas at resource level " + numReplicasAtResourceLevel
              + " different from number of replicas for partition " + partitionInstanceSet);
      ensureOrThrow(allPartitionsToInstancesInHelix.put(resourcePartition, partitionInstanceSet) == null,
          "Partition " + resourcePartition + " already found under a different resource.");
    }
  }
  for (PartitionId partitionId : partitionLayout.getPartitions()) {
    Partition partition = (Partition) partitionId;
    String partitionName = Long.toString(partition.getId());
    Set<String> replicaHostsInHelix = allPartitionsToInstancesInHelix.remove(partitionName);
    ensureOrThrow(replicaHostsInHelix != null, "No replicas found for partition " + partitionName + " in Helix");
    for (Replica replica : partition.getReplicas()) {
      if (replica.getDataNodeId().getDatacenterName().equals(dcName)) {
        String instanceName = getInstanceName(replica.getDataNodeId());
        ensureOrThrow(replicaHostsInHelix.remove(instanceName),
            "Instance " + instanceName + " for the given " + "replica in the clustermap not found in Helix");
      }
    }
    ensureOrThrow(replicaHostsInHelix.isEmpty(),
        "More instances in Helix than in clustermap for partition: " + partitionName + " additional instances: "
            + replicaHostsInHelix);
  }
  ensureOrThrow(allPartitionsToInstancesInHelix.isEmpty(),
      "More partitions in Helix than in clustermap, additional partitions: "
          + allPartitionsToInstancesInHelix.keySet());
}