Java IdealState.getInstanceStateMap方法代码示例

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Override
public boolean matches(Object o) {
  if (o == null || !(o instanceof IdealState)) {
    return false;
  }
  IdealState is = (IdealState)o;
  if (is.getRebalanceMode() != IdealState.RebalanceMode.CUSTOMIZED ||
      !is.getReplicas().equals("3") ||
      is.getNumPartitions() != this.numPartitions ||
      !is.getStateModelDefRef().equals("OnlineOffline")) {
    return false;
  }
  for (Map.Entry<Integer, List<String>> entry : partitionHostMap.entrySet()) {
    Map<String, String> stateMap = is.getInstanceStateMap(
            this.resource + "$" + entry.getKey());
    if (stateMap.size() != entry.getValue().size()) {
      return false;
    }
    for (String host : entry.getValue()) {
      if (!(stateMap.containsKey(host) && stateMap.get(host).equals("ONLINE"))) {
        return false;
      }
    }
  }
  return true;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private boolean hasInstanceMapChanged(Map<Partition, Map<String, String>> newAssiments,
    IdealState idealState) {
  Set<Partition> partitions = new HashSet<Partition>(newAssiments.keySet());
  for (String p : idealState.getPartitionSet()) {
    partitions.add(new Partition(p));
  }

  for (Partition partition : partitions) {
    Map<String, String> instanceMap = newAssiments.get(partition);
    Map<String, String> existInstanceMap =
        idealState.getInstanceStateMap(partition.getPartitionName());
    if (instanceMap == null && existInstanceMap == null) {
      continue;
    }
    if (instanceMap == null || existInstanceMap == null || !instanceMap
        .equals(existInstanceMap)) {
      return true;
    }
  }

  return false;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private void verifySemiAutoMasterSlaveAssignment(IdealState idealState) {
  for (String partition : idealState.getPartitionSet()) {
    Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition);
    List<String> preferenceList = idealState.getPreferenceList(partition);
    int numMaster = 0;

    for (String ins : preferenceList) {
      Assert.assertTrue(instanceStateMap.containsKey(ins),
          String.format("Instance %s from preference list not in the map", ins));
      String state = instanceStateMap.get(ins);
      Assert.assertTrue(state.equals(MasterSlaveSMD.States.MASTER.name()) || state
          .equals(MasterSlaveSMD.States.SLAVE.name()), "Actual State" + state);
      if (state.equals(MasterSlaveSMD.States.MASTER.name())) {
        numMaster++;
      }
    }

    Assert.assertEquals(numMaster, 1);
  }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private IdealState updateIdealState(IdealState idealState, int newNumReplicas) {
  idealState.setReplicas(Integer.toString(newNumReplicas));
  Set<String> segmentIds = idealState.getPartitionSet();
  for (String segmentId : segmentIds) {
    Map<String, String> instanceStateMap = idealState.getInstanceStateMap(segmentId);
    if (instanceStateMap.size() > newNumReplicas) {
      Set<String> keys = instanceStateMap.keySet();
      while (instanceStateMap.size() > newNumReplicas) {
        instanceStateMap.remove(keys.iterator().next());
      }
    } else if (instanceStateMap.size() < newNumReplicas) {
      throw new RuntimeException("Segment " + segmentId + " has " + instanceStateMap.size() +
          " replicas but want changed to " + newNumReplicas);
    }
  }
  return idealState;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private boolean shouldDeleteInProgressLLCSegment(final String segmentId, final IdealState idealState, RealtimeSegmentZKMetadata segmentZKMetadata) {
  if (idealState == null) {
    return false;
  }
  Map<String, String> stateMap = idealState.getInstanceStateMap(segmentId);
  if (stateMap == null) {
    // segment is there in propertystore but not in idealstate. mark for deletion
    return true;
  } else {
    Set<String> states = new HashSet<>(stateMap.values());
    if (states.size() == 1 && states
        .contains(CommonConstants.Helix.StateModel.SegmentOnlineOfflineStateModel.OFFLINE)) {
      // All replicas of this segment are offline, delete it if it is old enough
      final long now = System.currentTimeMillis();
      if (now - segmentZKMetadata.getCreationTime() >= TimeUnit.DAYS.toMillis(
          RETENTION_TIME_FOR_OLD_LLC_SEGMENTS_DAYS)) {
        return true;
      }
    }
  }
  return false;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private void updateIdealstateInfo(Map<String, Integer> topicPartitionMapForIdealState,
    IdealState idealStateForTopic) {
  for (String partition : idealStateForTopic.getPartitionSet()) {
    Map<String, String> idealStatesMap = idealStateForTopic.getInstanceStateMap(partition);
    for (String instance : idealStatesMap.keySet()) {
      if (!topicPartitionMapForIdealState.containsKey(instance)) {
        topicPartitionMapForIdealState.put(instance, 1);
      } else {
        topicPartitionMapForIdealState.put(instance,
            topicPartitionMapForIdealState.get(instance) + 1);
      }
    }
  }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
/**
 * validates that all partitions have same amount of data
 * @param sliceNumber
 * @return
 */
private boolean validate(int sliceNumber) {
  IdealState idealstate =
      _admin.getResourceIdealState(_clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME);
  String partitionName = MySQLConstants.MASTER_SLAVE_RESOURCE_NAME + "_" + sliceNumber;
  Map<String, String> instanceStateMap = idealstate.getInstanceStateMap(partitionName);
  Map<String, MasterStatus> masterStatusMap = new HashMap<String, MasterStatus>();
  for (String instance : instanceStateMap.keySet()) {
    InstanceConfig instanceConfig = _admin.getInstanceConfig(_clusterName, instance);
    MySQLAdmin admin = new MySQLAdmin(instanceConfig);
    MasterStatus masterStatus = admin.getMasterStatus();
    masterStatusMap.put(instance, masterStatus);
    admin.close();
  }
  for (String instance1 : instanceStateMap.keySet()) {
    MasterStatus masterStatus1 = masterStatusMap.get(instance1);
    String gtIdSet1 = masterStatus1.getString(MasterStatusAttribute.Executed_Gtid_Set);
    for (String instance2 : instanceStateMap.keySet()) {
      if (!instance1.equals(instance2)) {
        MasterStatus masterStatus2 = masterStatusMap.get(instance2);
        String gtIdSet2 = masterStatus2.getString(MasterStatusAttribute.Executed_Gtid_Set);
        if (!gtIdSet1.equals(gtIdSet2)) {
          LOG.error("Cluster is unhealthy: gtid set of " + instance1 + ":" + gtIdSet1
              + " does not match the gtid set of " + instance2 + ":" + gtIdSet2);
          return false;
        }
      }
    }
  }
  return true;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Override
public ResourceAssignment computeBestPossiblePartitionState(ClusterDataCache cache,
    IdealState idealState, Resource resource, CurrentStateOutput currentStateOutput) {
  // Looking for cached BestPossible mapping for this resource, if it is already there, do not recompute it again.
  // The cached mapping will be cleared in ClusterDataCache if there is anything changed in cluster state that can
  // cause the potential changes in BestPossible state.
  ResourceAssignment partitionMapping =
      cache.getCachedResourceAssignment(resource.getResourceName());
  if (partitionMapping != null) {
    return partitionMapping;
  }

  String stateModelDefName = idealState.getStateModelDefRef();
  StateModelDefinition stateModelDef = cache.getStateModelDef(stateModelDefName);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Processing resource:" + resource.getResourceName());
  }
  partitionMapping = new ResourceAssignment(resource.getResourceName());
  for (Partition partition : resource.getPartitions()) {
    Map<String, String> currentStateMap =
        currentStateOutput.getCurrentStateMap(resource.getResourceName(), partition);
    Set<String> disabledInstancesForPartition =
        cache.getDisabledInstancesForPartition(resource.getResourceName(), partition.toString());
    Map<String, String> idealStateMap =
        idealState.getInstanceStateMap(partition.getPartitionName());
    Map<String, String> bestStateForPartition =
        computeCustomizedBestStateForPartition(cache, stateModelDef, idealStateMap,
            currentStateMap, disabledInstancesForPartition, idealState.isEnabled());
    partitionMapping.addReplicaMap(partition, bestStateForPartition);
  }

  cache.setCachedResourceAssignment(resource.getResourceName(), partitionMapping);

  return partitionMapping;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private void verifyAssignmentInIdealStateWithPersistDisabled(IdealState idealState,
    Set<String> excludedInstances) {
  boolean mapFieldEmpty = true;
  boolean assignmentNotChanged = false;
  for (String partition : idealState.getPartitionSet()) {
    Map<String, String> instanceStateMap = idealState.getInstanceStateMap(partition);
    if (instanceStateMap == null || instanceStateMap.isEmpty()) {
      continue;
    }
    mapFieldEmpty = false;
    Set<String> instancesInMap = instanceStateMap.keySet();
    for (String ins : excludedInstances) {
      if(instancesInMap.contains(ins)) {
        // if at least one excluded instance is included, it means assignment was not updated.
        assignmentNotChanged = true;
      }
      if(idealState.getRebalanceMode() == RebalanceMode.FULL_AUTO) {
        List<String> instanceList = idealState.getPreferenceList(partition);
        if (instanceList.contains(ins)) {
          assignmentNotChanged = true;
        }
      }
    }
  }

  Assert.assertTrue((mapFieldEmpty || assignmentNotChanged),
      "BestPossible assignment was updated.");
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
/**
 * Validate instances for each partition is on different zone and with necessary tagged
 * instances.
 */
private void validateNoZeroReplica(IdealState is, ExternalView ev) {
  int replica = is.getReplicaCount(NUM_NODE);
  StateModelDefinition stateModelDef =
      BuiltInStateModelDefinitions.valueOf(is.getStateModelDefRef()).getStateModelDefinition();

  for (String partition : is.getPartitionSet()) {
    Map<String, String> evStateMap = ev.getRecord().getMapField(partition);
    Map<String, String> isStateMap = is.getInstanceStateMap(partition);
    validateMap(is.getResourceName(), partition, replica, evStateMap, stateModelDef);
    validateMap(is.getResourceName(), partition, replica, isStateMap, stateModelDef);
  }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private IdealState addLLCRealtimeSegmentsInIdealState(final IdealState idealState, Map<String, List<String>> idealStateEntries) {
  for (Map.Entry<String, List<String>> entry : idealStateEntries.entrySet()) {
    final String segmentId = entry.getKey();
    final Map<String, String> stateMap = idealState.getInstanceStateMap(segmentId);
    if (stateMap != null) {
      // Replace the segment if it already exists
      stateMap.clear();
    }
    for (String instanceName : entry.getValue()) {
      idealState.setPartitionState(segmentId, instanceName, PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
    }
  }
  return idealState;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Test
public void testUpdateHelixForSegmentClosing() throws Exception {
  final IdealState  idealState = PinotTableIdealStateBuilder.buildEmptyKafkaConsumerRealtimeIdealStateFor(
      "someTable_REALTIME", 17);
  final String s1 = "S1";
  final String s2 = "S2";
  final String s3 = "S3";
  String[] instanceArr = {s1, s2, s3};
  final String oldSegmentNameStr = "oldSeg";
  final String newSegmentNameStr = "newSeg";

  idealState.setPartitionState(oldSegmentNameStr, s1,
      PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
  idealState.setPartitionState(oldSegmentNameStr, s2,
      PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
  idealState.setPartitionState(oldSegmentNameStr, s3,
      PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE);
  PinotLLCRealtimeSegmentManager.updateForNewRealtimeSegment(idealState, Arrays.asList(instanceArr),
      oldSegmentNameStr, newSegmentNameStr);
  // Now verify that the old segment state is online in the idealstate and the new segment state is CONSUMING
  Map<String, String> oldsegStateMap = idealState.getInstanceStateMap(oldSegmentNameStr);
  Assert.assertEquals(oldsegStateMap.get(s1), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
  Assert.assertEquals(oldsegStateMap.get(s2), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
  Assert.assertEquals(oldsegStateMap.get(s3), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);

  Map<String, String> newsegStateMap = idealState.getInstanceStateMap(oldSegmentNameStr);
  Assert.assertEquals(oldsegStateMap.get(s1), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
  Assert.assertEquals(oldsegStateMap.get(s2), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
  Assert.assertEquals(oldsegStateMap.get(s3), PinotHelixSegmentOnlineOfflineStateModelGenerator.ONLINE_STATE);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public void run() throws Exception {

    startZookeeper();

    startController();

    startBroker();

    startServer();

    configureResources();

    uploadIndexSegments();

    final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress);
    Verifier customVerifier = new Verifier() {

      @Override
      public boolean verify() {
        List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
        LOGGER.info("Waiting for the cluster to be set up and indexes to be loaded on the servers"
            + new Timestamp(System.currentTimeMillis()));
        for (String resourceName : resourcesInCluster) {
          IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
          ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
          if (idealState == null || externalView == null) {
            return false;
          }
          Set<String> partitionSet = idealState.getPartitionSet();
          for (String partition : partitionSet) {
            Map<String, String> instanceStateMapIS = idealState.getInstanceStateMap(partition);
            Map<String, String> instanceStateMapEV = externalView.getStateMap(partition);
            if (instanceStateMapIS == null || instanceStateMapEV == null) {
              return false;
            }
            if (!instanceStateMapIS.equals(instanceStateMapEV)) {
              return false;
            }
          }
        }
        LOGGER.info("Cluster is ready to serve queries");
        return true;
      }
    };
    ClusterStateVerifier.verifyByPolling(customVerifier, 60 * 1000);

    postQueries();
  }
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public static void waitForExternalViewUpdate(String zkAddress, final String clusterName, long timeoutInMilliseconds) {
  final ZKHelixAdmin helixAdmin = new ZKHelixAdmin(zkAddress);

  /* Use one of these three classes instead of deprecated ClusterStateVerifier.
  BestPossibleExternalViewVerifier
  StrictMatchExternalViewVerifier
  ClusterLiveNodesVerifier
  */

  org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.Verifier customVerifier = new org.apache.helix.tools.ClusterVerifiers.ClusterStateVerifier.Verifier() {

    @Override
    public boolean verify() {
      List<String> resourcesInCluster = helixAdmin.getResourcesInCluster(clusterName);
      LOGGER.info("Waiting for the cluster to be set up and indexes to be loaded on the servers" + new Timestamp(
          System.currentTimeMillis()));
      for (String resourceName : resourcesInCluster) {
        // Only check table resources and broker resource
        if (!TableNameBuilder.isTableResource(resourceName) && !resourceName.equals(
            CommonConstants.Helix.BROKER_RESOURCE_INSTANCE)) {
          continue;
        }

        IdealState idealState = helixAdmin.getResourceIdealState(clusterName, resourceName);
        ExternalView externalView = helixAdmin.getResourceExternalView(clusterName, resourceName);
        if (idealState == null || externalView == null) {
          return false;
        }
        Set<String> partitionSet = idealState.getPartitionSet();
        for (String partition : partitionSet) {
          Map<String, String> instanceStateMapIS = idealState.getInstanceStateMap(partition);
          Map<String, String> instanceStateMapEV = externalView.getStateMap(partition);
          if (instanceStateMapIS == null || instanceStateMapEV == null) {
            return false;
          }
          if (!instanceStateMapIS.equals(instanceStateMapEV)) {
            return false;
          }
        }
      }
      LOGGER.info("Cluster is ready to serve queries");
      return true;
    }
  };

  ClusterStateVerifier.verifyByPolling(customVerifier, timeoutInMilliseconds);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
void validateLLCSegments(final String realtimeTableName, TableConfig tableConfig) {
  LOGGER.info("Validating LLC Segments for {}", realtimeTableName);
  Map<String, String> streamConfigs = tableConfig.getIndexingConfig().getStreamConfigs();
  ZNRecord partitionAssignment = _llcRealtimeSegmentManager.getKafkaPartitionAssignment(realtimeTableName);
  if (partitionAssignment == null) {
    LOGGER.warn("No partition assignment found for table {}", realtimeTableName);
    return;
  }
  Map<String, List<String>> partitionToHostsMap = partitionAssignment.getListFields();
  // Keep a set of kafka partitions, and remove the partition when we find a segment in CONSUMING state in
  // that partition.
  Set<Integer> nonConsumingKafkaPartitions = new HashSet<>(partitionToHostsMap.size());
  for (String partitionStr : partitionToHostsMap.keySet()) {
    nonConsumingKafkaPartitions.add(Integer.valueOf(partitionStr));
  }

  IdealState idealState =
      HelixHelper.getTableIdealState(_pinotHelixResourceManager.getHelixZkManager(), realtimeTableName);
  if (!idealState.isEnabled()) {
    // No validation to be done.
    LOGGER.info("Skipping validation for {} since it is disabled", realtimeTableName);
    return;
  }
  // Walk through all segments in the idealState, looking for one instance that is in CONSUMING state. If we find one
  // remove the kafka partition that the segment belongs to, from the kafka partition set.
  // Make sure that there are at least some LLC segments in place. If there are no LLC segments, it is possible
  // that this table is in the process of being disabled for LLC
  Set<String> segmentIds = idealState.getPartitionSet();
  List<String> llcSegments = new ArrayList<>(segmentIds.size());
  for (String segmentId : segmentIds) {
    if (SegmentName.isLowLevelConsumerSegmentName(segmentId)) {
      llcSegments.add(segmentId);
      Map<String, String> stateMap = idealState.getInstanceStateMap(segmentId);
      Iterator<String> iterator = stateMap.values().iterator();
      // If there is at least one instance in CONSUMING state, we are good.
      boolean foundConsuming = false;
      while (iterator.hasNext() && !foundConsuming) {
        String stateString = iterator.next();
        if (stateString.equals(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
          LOGGER.info("Found CONSUMING segment {}", segmentId);
          foundConsuming = true;
        }
      }
      if (foundConsuming) {
        LLCSegmentName llcSegmentName = new LLCSegmentName(segmentId);
        nonConsumingKafkaPartitions.remove(llcSegmentName.getPartitionId());
      }
    }
  }

  // Kafka partition set now has all the partitions that do not have any segments in CONSUMING state.
  if (!llcSegments.isEmpty()) {
    // Raise the metric only if there is at least one llc segment in the idealstate.
    _validationMetrics.updateNonConsumingPartitionCountMetric(realtimeTableName, nonConsumingKafkaPartitions.size());
    // Recreate a segment for the partitions that are missing one.
    for (Integer kafkaPartition : nonConsumingKafkaPartitions) {
      LOGGER.warn("Table {}, kafka partition {} has no segments in CONSUMING state (out of {} llc segments)",
          realtimeTableName, kafkaPartition, llcSegments.size());
    }
    if (_autoCreateOnError) {
      _llcRealtimeSegmentManager.createConsumingSegment(realtimeTableName, nonConsumingKafkaPartitions, llcSegments,
          tableConfig);
      _llcRealtimeSegmentManager.completeCommittingSegments(realtimeTableName, llcSegments);
    }
  }
  // Make this call after other validations (so that we verify that we are consistent against the existing partition
  // assignment). This call may end up changing the kafka partition assignment for the table.
  _llcRealtimeSegmentManager.updateKafkaPartitionsIfNecessary(tableConfig);
}