Java ExternalView.getPartitionSet方法代码示例

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private Set<String> getServingDataResource(List<ExternalView> externalViewList) {
  Set<String> servingDataResourceSet = new HashSet<String>();
  for (ExternalView externalView : externalViewList) {
    if (externalView.getResourceName().equals(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) {
      Set<String> dataResources = externalView.getPartitionSet();
      for (String dataResource : dataResources) {
        Map<String, String> dataResourceToServingBrokerMap = externalView.getStateMap(dataResource);
        if (dataResourceToServingBrokerMap.containsKey(_instanceId)
            && "ONLINE".equals(dataResourceToServingBrokerMap.get(_instanceId))) {
          servingDataResourceSet.add(dataResource);
        }
      }
    }
  }
  LOGGER.info("Current serving data resource : " + Arrays.toString(servingDataResourceSet.toArray(new String[0])));
  return servingDataResourceSet;
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private Map<String, Integer> getStateCountMap(ExternalView externalView) {
  Map<String, Integer> stateCountMap = new HashMap<String, Integer>();
  if (externalView == null) {
    return stateCountMap;
  }
  for (String slice : externalView.getPartitionSet()) {
    Map<String, String> stateMap = externalView.getStateMap(slice);
    for (String instance : stateMap.keySet()) {
      String state = stateMap.get(instance);
      if (!stateCountMap.containsKey(state)) {
        stateCountMap.put(state, 0);
      }
      stateCountMap.put(state, stateCountMap.get(state) + 1);
    }
  }
  return stateCountMap;
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private static void printState(String msg) {
  System.out.println("CLUSTER STATE: " + msg);
  ExternalView resourceExternalView = admin.getResourceExternalView(CLUSTER_NAME, RESOURCE_NAME);
  TreeSet<String> sortedSet = new TreeSet<String>(resourceExternalView.getPartitionSet());
  StringBuilder sb = new StringBuilder("\t\t");
  for (int i = 0; i < NUM_NODES; i++) {
    sb.append(INSTANCE_CONFIG_LIST.get(i).getInstanceName()).append("\t");
  }
  System.out.println(sb);
  for (String partitionName : sortedSet) {
    sb.delete(0, sb.length() - 1);
    sb.append(partitionName).append("\t");
    for (int i = 0; i < NUM_NODES; i++) {
      Map<String, String> stateMap = resourceExternalView.getStateMap(partitionName);
      if (stateMap != null && stateMap.containsKey(INSTANCE_CONFIG_LIST.get(i).getInstanceName())) {
        sb.append(stateMap.get(INSTANCE_CONFIG_LIST.get(i).getInstanceName()).charAt(0)).append(
            "\t\t");
      } else {
        sb.append("-").append("\t\t");
      }
    }
    System.out.println(sb);
  }
  System.out.println("###################################################################");
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
@Test
public void testCancellationWhenDisableResource() throws InterruptedException {
  // Enable cancellation
  ClusterConfig clusterConfig = _configAccessor.getClusterConfig(CLUSTER_NAME);
  clusterConfig.stateTransitionCancelEnabled(true);
  _configAccessor.setClusterConfig(CLUSTER_NAME, clusterConfig);

  // Wait for assignment done
  Thread.sleep(2000);

  // Disable the resource
  _setupTool.getClusterManagementTool()
      .enableResource(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB, false);


  // Wait for pipeline reaching final stage
  Thread.sleep(2000L);
  ExternalView externalView = _setupTool.getClusterManagementTool()
      .getResourceExternalView(CLUSTER_NAME, WorkflowGenerator.DEFAULT_TGT_DB);
  for (String partition : externalView.getPartitionSet()) {
    for (String currentState : externalView.getStateMap(partition).values()) {
      Assert.assertEquals(currentState, "OFFLINE");
    }
  }
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
@Test(dependsOnMethods = "testAddParticipant")
public void testStopAndReStartParticipant()
    throws InterruptedException {
  MockParticipantManager participant = _participants.get(0);
  String instance = participant.getInstanceName();

  Map<String, MasterSlaveSMD.States> affectedPartitions =
      new HashMap<String, MasterSlaveSMD.States>();

  ExternalView externalView = _accessor.getProperty(_keyBuilder.externalView(DB_NAME));

  for (String partition : externalView.getPartitionSet()) {
    Map<String, String> stateMap = externalView.getStateMap(partition);
    if (stateMap.containsKey(instance)) {
      affectedPartitions.put(partition, MasterSlaveSMD.States.valueOf(stateMap.get(instance)));
    }
  }

  stopParticipant(participant, affectedPartitions);

  // create a new participant
  participant = new MockParticipantManager(ZK_ADDR, CLUSTER_NAME, instance);
  _participants.set(0, participant);
  startParticipant(participant, affectedPartitions);
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private boolean isParentTaskDone(ExternalView ev) {
  Set<String> partitionSet = ev.getPartitionSet();
  if (partitionSet.isEmpty()) {
    return false;
  }

  for (String partition : partitionSet) {
    Map<String, String> stateMap = ev.getStateMap(partition);
    for (String instance : stateMap.keySet()) {
      if (!stateMap.get(instance).equalsIgnoreCase("Online")) {
        return false;
      }
    }
  }
  return true;
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private static void printStatus(HelixAdmin admin, String cluster, String resource) {
  ExternalView externalView = admin.getResourceExternalView(cluster, resource);
  // System.out.println(externalView);
  TreeSet<String> treeSet = new TreeSet<String>(externalView.getPartitionSet());
  System.out.println("lockName" + "\t" + "acquired By");
  System.out.println("======================================");
  for (String lockName : treeSet) {
    Map<String, String> stateMap = externalView.getStateMap(lockName);
    String acquiredBy = null;
    if (stateMap != null) {
      for (String instanceName : stateMap.keySet()) {
        if ("ONLINE".equals(stateMap.get(instanceName))) {
          acquiredBy = instanceName;
          break;
        }
      }
    }
    System.out.println(lockName + "\t" + ((acquiredBy != null) ? acquiredBy : "NONE"));
  }
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private List<InstanceConfig> getInstanceConfigs(ExternalView externalView) {
  List<InstanceConfig> instanceConfigs = new ArrayList<>();
  Set<String> instances = new HashSet<>();

  // Collect all unique instances
  for (String partitionName : externalView.getPartitionSet()) {
    for (String instance : externalView.getStateMap(partitionName).keySet()) {
      if (!instances.contains(instance)) {
        instanceConfigs.add(new InstanceConfig(instance));
        instances.add(instance);
      }
    }
  }

  return instanceConfigs;
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
private boolean isLargeCluster(ExternalView externalView) {
  // Check if the number of replicas is sufficient to treat it as a large cluster
  final String helixReplicaCount = externalView.getRecord().getSimpleField("REPLICAS");
  final int replicaCount;

  try {
    replicaCount = Integer.parseInt(helixReplicaCount);
  } catch (Exception e) {
    LOGGER.warn("Failed to parse the replica count ({}) from external view of table {}", helixReplicaCount,
        externalView.getResourceName());
    return false;
  }

  if (replicaCount < _minReplicaCountForLargeCluster) {
    return false;
  }

  // Check if the server count is high enough to count as a large cluster
  final Set<String> instanceSet = new HashSet<>();
  for (String partition : externalView.getPartitionSet()) {
    instanceSet.addAll(externalView.getStateMap(partition).keySet());
  }

  return _minServerCountForLargeCluster <= instanceSet.size();
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
public void init(ExternalView externalView, List<InstanceConfig> instanceConfigList) {
  RoutingTableInstancePruner instancePruner = new RoutingTableInstancePruner(instanceConfigList);
  for (String segmentName : externalView.getPartitionSet()) {
    List<String> serversForSegment = new ArrayList<>();
    for (Map.Entry<String, String> entry : externalView.getStateMap(segmentName).entrySet()) {
      String serverName = entry.getKey();
      if (entry.getValue().equals(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.ONLINE)
          && !instancePruner.isInactive(serverName)) {
        serversForSegment.add(serverName);
      }
    }
    if (!serversForSegment.isEmpty()) {
      _segmentToServersMap.put(segmentName, serversForSegment);
    }
  }
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
public ViewInfo(ExternalView externalView) {
  this.resource = externalView.getResourceName();
  Set<String> partitionSet = externalView.getPartitionSet();
  this.partitionMap = Maps.newTreeMap();
  for (String partition : partitionSet) {
    Map<String, String> stateMap = externalView.getStateMap(partition);
    Pair<String, Integer> resourceAndPartitionNum =
        TerrapinUtil.getResourceAndPartitionNum(partition);
    if (resourceAndPartitionNum == null) {
      LOG.warn("Invalid helix partition for " + resource + " : " + partition);
      continue;
    }
    if (stateMap == null) {
      continue;
    }
    List<String> instanceList = Lists.newArrayListWithCapacity(5);
    for (Map.Entry<String, String> entry : stateMap.entrySet()) {
      if (entry.getValue().equals("ONLINE")) {
        instanceList.add(entry.getKey());
      }
    }
    // Keep the list in alphabetical order.
    if (!instanceList.isEmpty()) {
      Collections.sort(instanceList);
      this.partitionMap.put(TerrapinUtil.getViewPartitionName(resource,
              resourceAndPartitionNum.getRight()), instanceList);
    }
  }
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
@Test
public void testRandomSegmentAssignmentStrategy() throws Exception {
  final int numReplicas = 2;

  // Adding table
  String OfflineTableConfigJson =
      ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(TABLE_NAME_RANDOM, null, null, numReplicas,
          "RandomAssignmentStrategy").toString();
  AbstractTableConfig offlineTableConfig = AbstractTableConfig.init(OfflineTableConfigJson);
  _pinotHelixResourceManager.addTable(offlineTableConfig);

  Thread.sleep(3000);
  for (int i = 0; i < 10; ++i) {
    addOneSegment(TABLE_NAME_RANDOM);
    Thread.sleep(2000);
    final Set<String> taggedInstances =
        _pinotHelixResourceManager.getAllInstancesForServerTenant("DefaultTenant_OFFLINE");
    final Map<String, Integer> instance2NumSegmentsMap = new HashMap<String, Integer>();
    for (final String instance : taggedInstances) {
      instance2NumSegmentsMap.put(instance, 0);
    }
    final ExternalView externalView =
        _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME,
            TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(TABLE_NAME_RANDOM));
    Assert.assertEquals(externalView.getPartitionSet().size(), i + 1);
    for (final String segmentId : externalView.getPartitionSet()) {
      Assert.assertEquals(externalView.getStateMap(segmentId).size(), numReplicas);
    }

  }
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
@Override
public List<ServerToSegmentSetMap> computeRoutingTableFromExternalView(String tableName, ExternalView externalView,
    List<InstanceConfig> instanceConfigList) {

  RoutingTableInstancePruner pruner = new RoutingTableInstancePruner(instanceConfigList);

  Set<String> segments = externalView.getPartitionSet();
  List<ServerToSegmentSetMap> routingTable = new ArrayList<ServerToSegmentSetMap>();
  Map<String, Map<String, Set<String>>> groupIdToRouting = new HashMap<String, Map<String, Set<String>>>();
  for (String segment : segments) {
    Map<String, String> instanceMap = externalView.getStateMap(segment);
    for (String instance : instanceMap.keySet()) {
      if (instanceMap.get(instance).equals(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.OFFLINE)
          || pruner.isShuttingDown(instance)) {
        continue;
      }

      String groupId = SegmentNameBuilder.Realtime.extractGroupIdName(segment);
      if (!groupIdToRouting.containsKey(groupId)) {
        groupIdToRouting.put(groupId, new HashMap<String, Set<String>>());
      }
      if (!groupIdToRouting.get(groupId).containsKey(instance)) {
        groupIdToRouting.get(groupId).put(instance, new HashSet<String>());
      }
      groupIdToRouting.get(groupId).get(instance).add(segment);
    }
  }
  for (Map<String, Set<String>> replicaRouting : groupIdToRouting.values()) {
    routingTable.add(new ServerToSegmentSetMap(replicaRouting));
  }
  return routingTable;
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
public synchronized void updateTimeBoundaryService(ExternalView externalView) {
  if (_propertyStore == null) {
    return;
  }
  String tableName = externalView.getResourceName();
  // Do nothing for realtime table.
  if (TableNameBuilder.getTableTypeFromTableName(tableName) == TableType.REALTIME) {
    return;
  }
  Set<String> offlineSegmentsServing = externalView.getPartitionSet();
  AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName);

  TimeUnit tableTimeUnit = getTimeUnitFromString(offlineTableConfig.getValidationConfig().getTimeType());
  if (!offlineSegmentsServing.isEmpty() && tableTimeUnit != null) {
    long maxTimeValue = -1;
    for (String segmentId : offlineSegmentsServing) {
      try {
        long endTime = -1;
        OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_propertyStore, tableName, segmentId);
        if (offlineSegmentZKMetadata.getEndTime() > 0) {
          if (offlineSegmentZKMetadata.getTimeUnit() != null) {
            endTime = tableTimeUnit.convert(offlineSegmentZKMetadata.getEndTime(), offlineSegmentZKMetadata.getTimeUnit());
          } else {
            endTime = offlineSegmentZKMetadata.getEndTime();
          }
        }
        maxTimeValue = Math.max(maxTimeValue, endTime);
      } catch (Exception e) {
        LOGGER.error("Error during convert end time for segment - " + segmentId + ", exceptions: " + e);
      }
    }
    TimeBoundaryInfo timeBoundaryInfo = new TimeBoundaryInfo();
    timeBoundaryInfo.setTimeColumn(offlineTableConfig.getValidationConfig().getTimeColumnName());
    timeBoundaryInfo.setTimeValue(Long.toString(maxTimeValue));
    _timeBoundaryInfoMap.put(tableName, timeBoundaryInfo);
  }
}
 

import org.apache.helix.model.ExternalView; //导入方法依赖的package包/类
/**
 * Returns the set of online instances from external view.
 *
 * @param resourceExternalView External view for the resource.
 * @return Set<String> of online instances in the external view for the resource.
 */
public static Set<String> getOnlineInstanceFromExternalView(ExternalView resourceExternalView) {
  Set<String> instanceSet = new HashSet<String>();
  if (resourceExternalView != null) {
    for (String partition : resourceExternalView.getPartitionSet()) {
      Map<String, String> stateMap = resourceExternalView.getStateMap(partition);
      for (String instance : stateMap.keySet()) {
        if (stateMap.get(instance).equalsIgnoreCase(ONLINE)) {
          instanceSet.add(instance);
        }
      }
    }
  }
  return instanceSet;
}