Java HelixAdmin.getInstancesInClusterWithTag方法代码示例

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Override
public List<String> getAssignedInstances(HelixAdmin helixAdmin, String helixClusterName,
    SegmentMetadata segmentMetadata, int numReplicas, String tenantName) {
  String serverTenantName = null;
  if ("realtime".equalsIgnoreCase(segmentMetadata.getIndexType())) {
    serverTenantName = ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(tenantName);
  } else {
    serverTenantName = ControllerTenantNameBuilder.getOfflineTenantNameForTenant(tenantName);
  }
  final Random random = new Random(System.currentTimeMillis());

  List<String> allInstanceList = helixAdmin.getInstancesInClusterWithTag(helixClusterName, serverTenantName);
  List<String> selectedInstanceList = new ArrayList<String>();
  for (int i = 0; i < numReplicas; ++i) {
    final int idx = random.nextInt(allInstanceList.size());
    selectedInstanceList.add(allInstanceList.get(idx));
    allInstanceList.remove(idx);
  }
  LOGGER.info("Segment assignment result for : " + segmentMetadata.getName() + ", in resource : "
      + segmentMetadata.getTableName() + ", selected instances: "
      + Arrays.toString(selectedInstanceList.toArray()));

  return selectedInstanceList;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static IdealState buildInitialRealtimeIdealStateFor(String realtimeTableName,
    AbstractTableConfig realtimeTableConfig, HelixAdmin helixAdmin, String helixClusterName,
    ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore) {
  KafkaStreamMetadata kafkaStreamMetadata =
      new KafkaStreamMetadata(realtimeTableConfig.getIndexingConfig().getStreamConfigs());
  String realtimeServerTenant =
      ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(realtimeTableConfig.getTenantConfig().getServer());
  switch (kafkaStreamMetadata.getConsumerType()) {
    case highLevel:
      IdealState idealState =
          buildInitialKafkaHighLevelConsumerRealtimeIdealStateFor(realtimeTableName, helixAdmin, helixClusterName,
              zkHelixPropertyStore);
      List<String> realtimeInstances = helixAdmin.getInstancesInClusterWithTag(helixClusterName, realtimeServerTenant);
      if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) {
        throw new RuntimeException("Number of instance in current tenant should be an integer multiples of the number of replications");
      }
      setupInstanceConfigForKafkaHighLevelConsumer(realtimeTableName, realtimeInstances.size(),
          Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()), realtimeTableConfig
              .getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances);
      return idealState;
    case simple:
    default:
      throw new UnsupportedOperationException("Not support kafka consumer type: "
          + kafkaStreamMetadata.getConsumerType());
  }
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static IdealState buildInitialHighLevelRealtimeIdealStateFor(String realtimeTableName,
    TableConfig realtimeTableConfig, HelixAdmin helixAdmin, String helixClusterName,
    ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore) {
  String realtimeServerTenant =
      ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(realtimeTableConfig.getTenantConfig().getServer());
  final List<String> realtimeInstances = helixAdmin.getInstancesInClusterWithTag(helixClusterName,
      realtimeServerTenant);
  IdealState idealState = buildEmptyKafkaConsumerRealtimeIdealStateFor(realtimeTableName, 1);
  if (realtimeInstances.size() % Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()) != 0) {
    throw new RuntimeException(
        "Number of instance in current tenant should be an integer multiples of the number of replications");
  }
  setupInstanceConfigForKafkaHighLevelConsumer(realtimeTableName, realtimeInstances.size(),
      Integer.parseInt(realtimeTableConfig.getValidationConfig().getReplication()),
      realtimeTableConfig.getIndexingConfig().getStreamConfigs(), zkHelixPropertyStore, realtimeInstances);
  return idealState;
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Override
public List<String> getAssignedInstances(HelixAdmin helixAdmin, String helixClusterName,
    SegmentMetadata segmentMetadata, int numReplicas, String tenantName) {
  String serverTenantName = null;
  if ("realtime".equalsIgnoreCase(segmentMetadata.getIndexType())) {
    serverTenantName = ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(tenantName);
  } else {
    serverTenantName = ControllerTenantNameBuilder.getOfflineTenantNameForTenant(tenantName);
  }

  List<String> allInstances =
      helixAdmin.getInstancesInClusterWithTag(helixClusterName, serverTenantName);
  List<String> selectedInstanceList = new ArrayList<String>();
  if (segmentMetadata.getShardingKey() != null) {
    for (String instance : allInstances) {
      if (HelixHelper.getInstanceConfigsMapFor(instance, helixClusterName, helixAdmin).get("shardingKey")
          .equalsIgnoreCase(segmentMetadata.getShardingKey())) {
        selectedInstanceList.add(instance);
      }
    }
    LOGGER.info("Segment assignment result for : " + segmentMetadata.getName() + ", in resource : "
        + segmentMetadata.getTableName() + ", selected instances: "
        + Arrays.toString(selectedInstanceList.toArray()));
    return selectedInstanceList;
  } else {
    throw new RuntimeException("Segment missing sharding key!");
  }
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
public static void buildLowLevelRealtimeIdealStateFor(String realtimeTableName, TableConfig realtimeTableConfig,
    HelixAdmin helixAdmin, String helixClusterName, IdealState idealState) {
  String realtimeServerTenant =
      ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(realtimeTableConfig.getTenantConfig().getServer());
  final List<String> realtimeInstances = helixAdmin.getInstancesInClusterWithTag(helixClusterName,
      realtimeServerTenant);
  boolean create = false;
  // Validate replicasPerPartition here.
  final String replicasPerPartitionStr = realtimeTableConfig.getValidationConfig().getReplicasPerPartition();
  if (replicasPerPartitionStr == null || replicasPerPartitionStr.isEmpty()) {
    throw new RuntimeException("Null or empty value for replicasPerPartition, expected a number");
  }
  final int nReplicas;
  try {
    nReplicas = Integer.valueOf(replicasPerPartitionStr);
  } catch (NumberFormatException e) {
    throw new PinotHelixResourceManager.InvalidTableConfigException(
        "Invalid value for replicasPerPartition, expected a number: " + replicasPerPartitionStr, e);
  }
  if (idealState == null) {
    idealState = buildEmptyKafkaConsumerRealtimeIdealStateFor(realtimeTableName, nReplicas);
    create = true;
  }
  LOGGER.info("Assigning partitions to instances for simple consumer for table {}", realtimeTableName);
  final KafkaStreamMetadata kafkaMetadata = new KafkaStreamMetadata(realtimeTableConfig.getIndexingConfig().getStreamConfigs());
  final PinotLLCRealtimeSegmentManager segmentManager = PinotLLCRealtimeSegmentManager.getInstance();
  final int nPartitions = getPartitionCount(kafkaMetadata);
  LOGGER.info("Assigning {} partitions to instances for simple consumer for table {}", nPartitions, realtimeTableName);
  segmentManager.setupHelixEntries(realtimeTableConfig, kafkaMetadata, nPartitions, realtimeInstances, idealState, create);
}
 

import org.apache.helix.HelixAdmin; //导入方法依赖的package包/类
@Override
public List<String> getAssignedInstances(HelixAdmin helixAdmin, String helixClusterName,
    SegmentMetadata segmentMetadata, int numReplicas, String tenantName) {
  String serverTenantName;
  String tableName;
  if ("realtime".equalsIgnoreCase(segmentMetadata.getIndexType())) {
    tableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(segmentMetadata.getTableName());
    serverTenantName = ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(tenantName);
  } else {
    tableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(segmentMetadata.getTableName());
    serverTenantName = ControllerTenantNameBuilder.getOfflineTenantNameForTenant(tenantName);
  }

  List<String> selectedInstances = new ArrayList<String>();
  Map<String, Integer> currentNumSegmentsPerInstanceMap = new HashMap<String, Integer>();
  List<String> allTaggedInstances = helixAdmin.getInstancesInClusterWithTag(helixClusterName, serverTenantName);
  for (String instance : allTaggedInstances) {
    currentNumSegmentsPerInstanceMap.put(instance, 0);
  }
  ExternalView externalView = helixAdmin.getResourceExternalView(helixClusterName, tableName);
  if (externalView != null) {
    for (String partitionName : externalView.getPartitionSet()) {
      Map<String, String> instanceToStateMap = externalView.getStateMap(partitionName);
      for (String instanceName : instanceToStateMap.keySet()) {
        if (currentNumSegmentsPerInstanceMap.containsKey(instanceName)) {
          currentNumSegmentsPerInstanceMap.put(instanceName, currentNumSegmentsPerInstanceMap.get(instanceName) + 1);
        } else {
          currentNumSegmentsPerInstanceMap.put(instanceName, 1);
        }
      }
    }

  }
  PriorityQueue<Number2ObjectPair<String>> priorityQueue =
      new PriorityQueue<Number2ObjectPair<String>>(numReplicas, Pairs.getDescendingnumber2ObjectPairComparator());
  for (String key : currentNumSegmentsPerInstanceMap.keySet()) {
    priorityQueue.add(new Number2ObjectPair<String>(currentNumSegmentsPerInstanceMap.get(key), key));
    if (priorityQueue.size() > numReplicas) {
      priorityQueue.poll();
    }
  }

  while (!priorityQueue.isEmpty()) {
    selectedInstances.add(priorityQueue.poll().getB());
  }
  LOGGER.info("Segment assignment result for : " + segmentMetadata.getName() + ", in resource : "
      + segmentMetadata.getTableName() + ", selected instances: " + Arrays.toString(selectedInstances.toArray()));
  return selectedInstances;
}