Java IdealState.getNumPartitions方法代码示例

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) {
  Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    int numPartitions = is.getNumPartitions();
    for (int partition = 0; partition < numPartitions; ++partition) {
      if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) {
        TopicPartition tpi = new TopicPartition(topic, partition);
        unassignedPartitions.add(tpi);
      }
    }
  }
  return unassignedPartitions;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Override
public boolean matches(Object o) {
  if (o == null || !(o instanceof IdealState)) {
    return false;
  }
  IdealState is = (IdealState)o;
  if (is.getRebalanceMode() != IdealState.RebalanceMode.CUSTOMIZED ||
      !is.getReplicas().equals("3") ||
      is.getNumPartitions() != this.numPartitions ||
      !is.getStateModelDefRef().equals("OnlineOffline")) {
    return false;
  }
  for (Map.Entry<Integer, List<String>> entry : partitionHostMap.entrySet()) {
    Map<String, String> stateMap = is.getInstanceStateMap(
            this.resource + "$" + entry.getKey());
    if (stateMap.size() != entry.getValue().size()) {
      return false;
    }
    for (String host : entry.getValue()) {
      if (!(stateMap.containsKey(host) && stateMap.get(host).equals("ONLINE"))) {
        return false;
      }
    }
  }
  return true;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public static PistachioClusterInfo getClusterInfo() {
    try {
        String zookeeperConnStr = ConfigurationManager.getConfiguration().getString("Pistachio.ZooKeeper.Server");
        ZKHelixAdmin admin = new ZKHelixAdmin(zookeeperConnStr);
        IdealState idealState = admin.getResourceIdealState("PistachiosCluster", "PistachiosResource");
        PistachioClusterInfo info = new PistachioClusterInfo();
        info.numPartitions = idealState.getNumPartitions();
        info.numReplicas = Integer.parseInt(idealState.getReplicas());
        info.hostList = admin.getInstancesInCluster("PistachiosCluster");

        logger.info("num partitions: {}, num Replicas: {}, hostList: {}.", info.numPartitions,
            info.numReplicas, Joiner.on(",").join(info.hostList.toArray()));

        return info;
    } catch (Exception e) {
        logger.info("error getting cluster info", e);
        return null;
    }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public static IdealState expandCustomRebalanceModeIdealStateFor(IdealState oldIdealState,
    String topicName, int newNumTopicPartitions,
    PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(newNumTopicPartitions).setNumReplica(1)
      .setMaxPartitionsPerNode(newNumTopicPartitions);

  int numOldPartitions = oldIdealState.getNumPartitions();
  for (int i = 0; i < numOldPartitions; ++i) {
    String partitionName = Integer.toString(i);
    try {
      String instanceName =
          oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next();
      customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
    } catch (Exception e) {
      // No worker added into the cluster.
    }
  }
  for (int i = numOldPartitions; i < newNumTopicPartitions; ++i) {
    InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll();
    customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i),
        liveInstance.getInstanceName(), "ONLINE");
    liveInstance.addTopicPartition(new TopicPartition(topicName, i));
    instanceToNumServingTopicPartitionMap.add(liveInstance);
  }
  return customModeIdealStateBuilder.build();
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public long getTotalPartition(String resource) {
    if (totalParition == -1) {
        synchronized(totalParition) {
            if (totalParition == -1) {
                ZKHelixAdmin admin = new ZKHelixAdmin(zkAddress);
                IdealState idealState = admin.getResourceIdealState(helixClusterName, resource);
                totalParition = (long)idealState.getNumPartitions();
            }
        }
    }

    return totalParition;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public void createTable(String clusterName, String dbName, String table, String tableSpec) {
  LOG.info("Creating table:" + table + " with table_spec: " + tableSpec + " in dbName:" + dbName);
  IdealState dbIdealState = _helixAdmin.getResourceIdealState(clusterName, dbName);
  int numPartitions = dbIdealState.getNumPartitions();
  String dbTableName = dbName + "." + table;
  AutoModeISBuilder builder = new AutoModeISBuilder(dbTableName);
  builder.setRebalancerMode(RebalanceMode.SEMI_AUTO);
  builder.setNumPartitions(numPartitions);
  builder.setNumReplica(Integer.parseInt(dbIdealState.getReplicas()));
  builder.setStateModel("OnlineOffline");
  builder.setStateModelFactoryName("TableTransitionHandlerFactory");
  for (String dbPartitionName : dbIdealState.getPartitionSet()) {
    String tablePartitionName = dbPartitionName + "." + table;
    Set<String> instanceSet = dbIdealState.getInstanceSet(dbPartitionName);
    builder.add(tablePartitionName);
    String[] instanceNames = new String[instanceSet.size()];
    instanceSet.toArray(instanceNames);
    builder.assignPreferenceList(tablePartitionName, instanceNames);
  }
  IdealState idealState = builder.build();

  // before setting the idealstate, set the configuration
  Map<String, String> properties = new HashMap<String, String>();
  properties.put("table_spec", tableSpec);
  properties.put("type", "TABLE");

  HelixConfigScope scope =
      new HelixConfigScopeBuilder(ConfigScopeProperty.RESOURCE).forCluster(clusterName)
          .forResource(dbTableName).build();
  _helixAdmin.setConfig(scope, properties);
  _helixAdmin.setResourceIdealState(clusterName, dbTableName, idealState);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public synchronized String validateExternalView() {
  try {
    Map<String, Integer> topicPartitionMapForIdealState =
        new HashMap<String, Integer>();
    Map<String, Integer> topicPartitionMapForExternalView =
        new HashMap<String, Integer>();
    int numOnlineTopicPartitions = 0;
    int numOfflineTopicPartitions = 0;
    int numErrorTopicPartitions = 0;
    int numTopicPartitions = 0;
    int numServingTopics = 0;
    int numErrorTopics = 0;
    for (String topicName : _helixMirrorMakerManager.getTopicLists()) {
      numServingTopics++;
      IdealState idealStateForTopic =
          _helixMirrorMakerManager.getIdealStateForTopic(topicName);
      ExternalView externalViewForTopic =
          _helixMirrorMakerManager.getExternalViewForTopic(topicName);
      numTopicPartitions += idealStateForTopic.getNumPartitions();
      if (idealStateForTopic.getNumPartitions() != externalViewForTopic.getPartitionSet()
          .size()) {
        numErrorTopics++;
        LOGGER.error(
            "For topic:{}, number of partitions for IdealState: {} doesn't match ExternalView: {}",
            topicName, idealStateForTopic, externalViewForTopic);
      }

      // IdealState Counting
      updateIdealstateInfo(topicPartitionMapForIdealState, idealStateForTopic);
      // ExternalView Counting
      for (String partition : externalViewForTopic.getPartitionSet()) {
        Map<String, String> stateMap = externalViewForTopic.getStateMap(partition);
        for (String instance : stateMap.keySet()) {
          String state = stateMap.get(instance);
          if (!topicPartitionMapForExternalView.containsKey(instance)) {
            topicPartitionMapForExternalView.put(instance, 1);
          } else {
            topicPartitionMapForExternalView.put(instance,
                topicPartitionMapForExternalView.get(instance) + 1);
          }
          if ("ONLINE".equalsIgnoreCase(state)) {
            numOnlineTopicPartitions++;
          } else if ("OFFLINE".equalsIgnoreCase(state)) {
            numOfflineTopicPartitions++;
          } else if ("ERROR".equalsIgnoreCase(state)) {
            numErrorTopicPartitions++;
          }
        }
      }
    }

    if (_helixMirrorMakerManager.isLeader()) {
      updateMetrics(numOnlineTopicPartitions, numOfflineTopicPartitions, numErrorTopicPartitions,
          numTopicPartitions, numServingTopics, numErrorTopics);
      updatePerWorkerISMetrics(topicPartitionMapForExternalView);
      updatePerWorkerEVMetrics(topicPartitionMapForExternalView);
    }
    JSONObject perWorkerISCounterJson =
        constructPerWorkerISCounterJson(topicPartitionMapForIdealState);
    JSONObject perWorkerEVCounterJson =
        constructPerWorkerEVCounterJson(topicPartitionMapForExternalView);
    JSONObject validationResultJson = constructValidationResultJson(numOnlineTopicPartitions,
        numOfflineTopicPartitions, numErrorTopicPartitions, numTopicPartitions,
        numServingTopics, numErrorTopics, perWorkerISCounterJson, perWorkerEVCounterJson);
    return validationResultJson.toJSONString();
  } catch (Exception e) {
    JSONObject jsonObject = new JSONObject();
    jsonObject.put("excpetion", e);
    return jsonObject.toJSONString();
  }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public boolean init() {

        boolean initialized = false;

        logger.info("Initializing profile server...........");
        logger.info("do nothing setting {}", doNothing);
        try {
            // open profile store
            Configuration conf = ConfigurationManager.getConfiguration();
        ZKHelixAdmin admin = new ZKHelixAdmin(conf.getString(ZOOKEEPER_SERVER));
        IdealState idealState = admin.getResourceIdealState("PistachiosCluster", "PistachiosResource");
        long totalParition = (long)idealState.getNumPartitions();
            profileStore = new LocalStorageEngine(
                    conf.getString(PROFILE_BASE_DIR),
                    (int)totalParition, 8,
                    conf.getInt("StorageEngine.KC.RecordsPerPartition"),
                    conf.getLong("StorageEngine.KC.MemoryPerPartition"));
            ProcessorRegistry.getInstance().init();
            logger.info("creating helix partition sepctator {} {} {}", conf.getString(ZOOKEEPER_SERVER, "EMPTY"),
                    "PistachiosCluster", conf.getString(PROFILE_HELIX_INSTANCE_ID, "EMPTY"));
            helixPartitionSpectator = HelixPartitionSpectator.getInstance(
                    conf.getString(ZOOKEEPER_SERVER), // zkAddr
                    "PistachiosCluster",
                    NativeUtils.getHostname() //InetAddress.getLocalHost().getHostName() //conf.getString(PROFILE_HELIX_INSTANCE_ID) // instanceName
                    );
            // Partition Manager for line spending
            manager = new HelixPartitionManager<>(
                    conf.getString(ZOOKEEPER_SERVER), // zkAddr
                    "PistachiosCluster",
                    NativeUtils.getHostname() //InetAddress.getLocalHost().getHostName() //conf.getString(PROFILE_HELIX_INSTANCE_ID) // instanceName
                    );
            //manager.start("BootstrapOnlineOffline", new BootstrapOnlineOfflineStateModelFactory(new StorePartitionHandlerFactory()));
            manager.start("MasterSlave", new BootstrapOnlineOfflineStateModelFactory(new StorePartitionHandlerFactory()));
            //            }

            initialized = true;
        } catch (Exception e) {
            logger.error("Failed to initialize ProfileServerModule", e);
        }
        logger.info("Finished initializing profile server...........");

        return initialized;
    }
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public void run() throws Exception {
  LOG.info("Starting load generator");
  String resourceName = MySQLConstants.MASTER_SLAVE_RESOURCE_NAME;
  IdealState idealState = _admin.getResourceIdealState(_clusterName, resourceName);
  int numSlices = idealState.getNumPartitions();
  boolean isHealthy = true;
  new Thread(_generator).start();
  _generator.pause();
  int iter = 0;

  while (isHealthy) {
    LOG.info("Starting iteration:" + iter++);
    LOG.info("Un-Pausing LoadGenerator");
    _generator.unpause();
    // generate load for a minute
    Thread.sleep(10 * 1000);
    // pick a random slice and disable the master
    int randomSlice = (int) (Math.random() * 1000) % numSlices;
    LOG.info("Randomly selected sliceNum:" + randomSlice + " to simulate failure");
    String currentMaster = null;
    currentMaster = getCurrentMaster(randomSlice);
    LOG.info("Current master for sliceNum:" + randomSlice + " is " + currentMaster);
    if (currentMaster != null) {
      LOG.info("Pausing LoadGenerator");
      _generator.pause();
      String oldMaster = currentMaster;
      // disable the current master
      LOG.info("Disabling the current master:" + currentMaster);
      _admin.enableInstance(_clusterName, currentMaster, false);
      String newMaster = null;
      // wait until there is a new master
      LOG.info("Waiting for new master:");
      do {
        Thread.sleep(1000);
        newMaster = getCurrentMaster(randomSlice);
      } while (newMaster == null || newMaster.equals(oldMaster));
      LOG.info("New master:" + newMaster);
      // start writes on new master
      _generator.unpause();
      // wait for writes to happen on the new master for X seconds
      Thread.sleep(10 * 1000);
      // enable the old master
      LOG.info("Enabling the old master:" + currentMaster);
      _admin.enableInstance(_clusterName, oldMaster, true);
      _generator.pause();
      // wait for it to become slave and catch up
      Thread.sleep(10000);
      LOG.info("Validating cluster health");
      isHealthy = validate(randomSlice);
    } else {
      LOG.error("No master available for slice:" + randomSlice);
      break;
    }
  }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
void rebalance(String clusterName, String resourceName, int replica, String keyPrefix,
    List<String> instanceNames, String groupId) {
  // ensure we get the same idealState with the same set of instances
  Collections.sort(instanceNames);

  IdealState idealState = getResourceIdealState(clusterName, resourceName);
  if (idealState == null) {
    throw new HelixException("Resource: " + resourceName + " has NOT been added yet");
  }

  if (groupId != null && groupId.length() > 0) {
    idealState.setInstanceGroupTag(groupId);
  }
  idealState.setReplicas(Integer.toString(replica));
  int partitions = idealState.getNumPartitions();
  String stateModelName = idealState.getStateModelDefRef();
  StateModelDefinition stateModDef = getStateModelDef(clusterName, stateModelName);

  if (stateModDef == null) {
    throw new HelixException("cannot find state model: " + stateModelName);
  }
  // StateModelDefinition def = new StateModelDefinition(stateModDef);

  List<String> statePriorityList = stateModDef.getStatesPriorityList();

  String masterStateValue = null;
  String slaveStateValue = null;
  replica--;

  for (String state : statePriorityList) {
    String count = stateModDef.getNumInstancesPerState(state);
    if (count.equals("1")) {
      if (masterStateValue != null) {
        throw new HelixException("Invalid or unsupported state model definition");
      }
      masterStateValue = state;
    } else if (count.equalsIgnoreCase("R")) {
      if (slaveStateValue != null) {
        throw new HelixException("Invalid or unsupported state model definition");
      }
      slaveStateValue = state;
    } else if (count.equalsIgnoreCase("N")) {
      if (!(masterStateValue == null && slaveStateValue == null)) {
        throw new HelixException("Invalid or unsupported state model definition");
      }
      replica = instanceNames.size() - 1;
      masterStateValue = slaveStateValue = state;
    }
  }
  if (masterStateValue == null && slaveStateValue == null) {
    throw new HelixException("Invalid or unsupported state model definition");
  }

  if (masterStateValue == null) {
    masterStateValue = slaveStateValue;
  }
  if (idealState.getRebalanceMode() != RebalanceMode.FULL_AUTO
      && idealState.getRebalanceMode() != RebalanceMode.USER_DEFINED) {
    ZNRecord newIdealState = DefaultIdealStateCalculator
        .calculateIdealState(instanceNames, partitions, replica, keyPrefix, masterStateValue,
            slaveStateValue);

    // for now keep mapField in SEMI_AUTO mode and remove listField in CUSTOMIZED mode
    if (idealState.getRebalanceMode() == RebalanceMode.SEMI_AUTO) {
      idealState.getRecord().setListFields(newIdealState.getListFields());
      // TODO: need consider to remove this.
      idealState.getRecord().setMapFields(newIdealState.getMapFields());
    }
    if (idealState.getRebalanceMode() == RebalanceMode.CUSTOMIZED) {
      idealState.getRecord().setMapFields(newIdealState.getMapFields());
    }
  } else {
    for (int i = 0; i < partitions; i++) {
      String partitionName = keyPrefix + "_" + i;
      idealState.getRecord().setMapField(partitionName, new HashMap<String, String>());
      idealState.getRecord().setListField(partitionName, new ArrayList<String>());
    }
  }
  setResourceIdealState(clusterName, resourceName, idealState);
}