Java IdealState.setNumPartitions方法代码示例

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
/**
 * Add the new specified segment to the idealState of the specified table in the specified cluster.
 *
 * @param helixManager The HelixManager object to access the helix cluster.
 * @param tableName Name of the table to which the new segment is to be added.
 * @param segmentName Name of the new segment to be added
 * @param getInstancesForSegment Callable returning list of instances where the segment should be uploaded.
 */
public static void addSegmentToIdealState(HelixManager helixManager, String tableName, final String segmentName,
    final Callable<List<String>> getInstancesForSegment) {

  Function<IdealState, IdealState> updater = new Function<IdealState, IdealState>() {
    @Override
    public IdealState apply(IdealState idealState) {
      List<String> targetInstances = null;
      try {
        targetInstances = getInstancesForSegment.call();
      } catch (Exception e) {
        LOGGER.error("Unable to get new instances for segment uploading.");
        return null;
      }

      for (final String instance : targetInstances) {
        idealState.setPartitionState(segmentName, instance, ONLINE);
      }

      idealState.setNumPartitions(idealState.getNumPartitions() + 1);
      return idealState;
    }
  };

  updateIdealState(helixManager, tableName, updater, DEFAULT_RETRY_POLICY);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public IdealState doInitialAssignment(String clusterName, List<String> instanceNames,
    int replicationFactor) {
  IdealState idealState = new IdealState(MySQLConstants.MASTER_SLAVE_RESOURCE_NAME);
  idealState.setStateModelDefRef("MasterSlave");
  idealState.setRebalanceMode(RebalanceMode.CUSTOMIZED);
  if (instanceNames.size() % replicationFactor != 0) {
    LOG.error(String.format(
        "Number of instances (%s) in the cluster must be a multiple of replication factor (%s)",
        instanceNames.size(), replicationFactor));
    return null;
  }
  int numSlices = instanceNames.size() / replicationFactor;
  idealState.setNumPartitions(numSlices);
  idealState.setReplicas(String.valueOf(replicationFactor));
  Collections.sort(instanceNames);
  for (int i = 0; i < numSlices; i++) {
    for (int j = 0; j < replicationFactor; j++) {
      idealState.setPartitionState(MySQLConstants.MASTER_SLAVE_RESOURCE_NAME + "_" + i,
          instanceNames.get(i * replicationFactor + j), (j == 0) ? "MASTER" : "SLAVE");
    }
  }
  LOG.info("Creating initial assignment \n" + idealState);
  _helixAdmin.setResourceIdealState(clusterName, MySQLConstants.MASTER_SLAVE_RESOURCE_NAME,
      idealState);
  return idealState;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Override
public void addResource(String clusterName, String resourceName, int partitions,
    String stateModelRef, String rebalancerMode, String rebalanceStrategy, int bucketSize,
    int maxPartitionsPerInstance) {
  if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
    throw new HelixException("cluster " + clusterName + " is not setup yet");
  }

  IdealState idealState = new IdealState(resourceName);
  idealState.setNumPartitions(partitions);
  idealState.setStateModelDefRef(stateModelRef);
  RebalanceMode mode =
      idealState.rebalanceModeFromString(rebalancerMode, RebalanceMode.SEMI_AUTO);
  idealState.setRebalanceMode(mode);
  idealState.setRebalanceStrategy(rebalanceStrategy);
  idealState.setReplicas("" + 0);
  idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);
  if (maxPartitionsPerInstance > 0 && maxPartitionsPerInstance < Integer.MAX_VALUE) {
    idealState.setMaxPartitionsPerInstance(maxPartitionsPerInstance);
  }
  if (bucketSize > 0) {
    idealState.setBucketSize(bucketSize);
  }
  addResource(clusterName, resourceName, idealState);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
/**
 * Create an IdealState for a resource that belongs to a resource group We use
 * "resourceGroupName$resourceInstanceTag" as the IdealState znode name to differetiate different
 * resources from the same resourceGroup.
 */
public IdealState createIdealStateForResourceGroup(String resourceGroupName,
    String resourceTag, int numPartition, int replica, String rebalanceMode, String stateModelDefName) {
  String idealStateId = genIdealStateNameWithResourceTag(resourceGroupName, resourceTag);
  IdealState idealState = new IdealState(idealStateId);
  idealState.setNumPartitions(numPartition);
  idealState.setStateModelDefRef(stateModelDefName);
  IdealState.RebalanceMode mode =
      idealState.rebalanceModeFromString(rebalanceMode, IdealState.RebalanceMode.SEMI_AUTO);
  idealState.setRebalanceMode(mode);
  idealState.setReplicas("" + replica);
  idealState.setStateModelFactoryName(HelixConstants.DEFAULT_STATE_MODEL_FACTORY);
  idealState.setResourceGroupName(resourceGroupName);
  idealState.setInstanceGroupTag(resourceTag);
  idealState.enableGroupRouting(true);

  return idealState;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Test(dataProvider = "IdealStateInput")
public void testIdealStateAssignment(String clusterName,
    List<String> instances, List<String> partitions, String numReplicas, String stateModeDef,
    String strategyName, Map<String, Map<String, String>> expectedMapping)
    throws IllegalAccessException, InstantiationException, ClassNotFoundException {
  ClusterConfig clusterConfig = new ClusterConfig(clusterName);
  List<InstanceConfig> instanceConfigs = new ArrayList<>();
  for (String instance : instances) {
    instanceConfigs.add(new InstanceConfig(instance));
  }

  IdealState idealState = new IdealState("TestResource");
  idealState.setStateModelDefRef(stateModeDef);
  idealState.setNumPartitions(partitions.size());
  idealState.setReplicas(numReplicas);

  Map<String, Map<String, String>> idealStateMapping = HelixUtil
      .getIdealAssignmentForFullAuto(clusterConfig, instanceConfigs, instances, idealState,
          partitions, strategyName);
  Assert.assertEquals(idealStateMapping, expectedMapping);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private void updateConfigs() throws InterruptedException {
  final Random r = new Random(System.currentTimeMillis());
  // test change content
  HelixDataAccessor accessor = _manager.getHelixDataAccessor();
  Builder keyBuilder = accessor.keyBuilder();
  final List<String> instances = accessor.getChildNames(keyBuilder.instanceConfigs());
  for (String instance : instances) {
    InstanceConfig value = accessor.getProperty(keyBuilder.instanceConfig(instance));
    value._record.setLongField("TimeStamp", System.currentTimeMillis());
    accessor.setProperty(keyBuilder.instanceConfig(instance), value);
    Thread.sleep(50);
  }

  final List<String> resources = accessor.getChildNames(keyBuilder.idealStates());
  for (String resource : resources) {
    IdealState idealState = accessor.getProperty(keyBuilder.idealStates(resource));
    idealState.setNumPartitions(r.nextInt(100));
    accessor.setProperty(keyBuilder.idealStates(idealState.getId()), idealState);
    Thread.sleep(20); // wait zk callback
  }
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public static Map<String, IdealState> getIdealStatesFromAssignment(
    Set<InstanceTopicPartitionHolder> newAssignment) {
  Map<String, CustomModeISBuilder> idealStatesBuilderMap =
      new HashMap<String, CustomModeISBuilder>();
  for (InstanceTopicPartitionHolder instance : newAssignment) {
    for (TopicPartition tpi : instance.getServingTopicPartitionSet()) {
      String topicName = tpi.getTopic();
      String partition = Integer.toString(tpi.getPartition());
      if (!idealStatesBuilderMap.containsKey(topicName)) {
        final CustomModeISBuilder customModeIdealStateBuilder =
            new CustomModeISBuilder(topicName);
        customModeIdealStateBuilder
            .setStateModel(OnlineOfflineStateModel.name)
            .setNumReplica(1);

        idealStatesBuilderMap.put(topicName, customModeIdealStateBuilder);
      }
      idealStatesBuilderMap.get(topicName).assignInstanceAndState(partition,
          instance.getInstanceName(),
          "ONLINE");
    }
  }
  Map<String, IdealState> idealStatesMap = new HashMap<String, IdealState>();
  for (String topic : idealStatesBuilderMap.keySet()) {
    IdealState idealState = idealStatesBuilderMap.get(topic).build();
    idealState.setMaxPartitionsPerInstance(idealState.getPartitionSet().size());
    idealState.setNumPartitions(idealState.getPartitionSet().size());
    idealStatesMap.put(topic, idealState);
  }
  return idealStatesMap;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
@Override
public void addClusterToGrandCluster(String clusterName, String grandCluster) {
  if (!ZKUtil.isClusterSetup(grandCluster, _zkClient)) {
    throw new HelixException("Grand cluster " + grandCluster + " is not setup yet");
  }

  if (!ZKUtil.isClusterSetup(clusterName, _zkClient)) {
    throw new HelixException("Cluster " + clusterName + " is not setup yet");
  }

  IdealState idealState = new IdealState(clusterName);

  idealState.setNumPartitions(1);
  idealState.setStateModelDefRef("LeaderStandby");

  List<String> controllers = getInstancesInCluster(grandCluster);
  if (controllers.size() == 0) {
    throw new HelixException("Grand cluster " + grandCluster + " has no instances");
  }
  idealState.setReplicas(Integer.toString(controllers.size()));
  Collections.shuffle(controllers);
  idealState.getRecord().setListField(clusterName, controllers);
  idealState.setPartitionState(clusterName, controllers.get(0), "LEADER");
  for (int i = 1; i < controllers.size(); i++) {
    idealState.setPartitionState(clusterName, controllers.get(i), "STANDBY");
  }

  ZKHelixDataAccessor accessor =
      new ZKHelixDataAccessor(grandCluster, new ZkBaseDataAccessor<ZNRecord>(_zkClient));
  Builder keyBuilder = accessor.keyBuilder();

  accessor.setProperty(keyBuilder.idealStates(idealState.getResourceName()), idealState);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
protected List<IdealState> setupIdealState(int nodes, String[] resources, int partitions,
    int replicas, RebalanceMode rebalanceMode, String stateModelName, String rebalanceClassName,
    String rebalanceStrategyName) {
  List<IdealState> idealStates = new ArrayList<IdealState>();
  for (int i = 0; i < resources.length; i++) {
    String resourceName = resources[i];
    ZNRecord record = new ZNRecord(resourceName);
    for (int p = 0; p < partitions; p++) {
      List<String> value = new ArrayList<String>();
      for (int r = 0; r < replicas; r++) {
        value.add(HOSTNAME_PREFIX + (p + r + 1) % nodes);
      }
      record.setListField(resourceName + "_" + p, value);
    }
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef(stateModelName);
    if (rebalanceClassName != null) {
      idealState.setRebalancerClassName(rebalanceClassName);
    }
    if (rebalanceStrategyName != null) {
      idealState.setRebalanceStrategy(rebalanceStrategyName);
    }
    idealState.setRebalanceMode(rebalanceMode);
    idealState.setNumPartitions(partitions);
    idealStates.add(idealState);
    idealState.setReplicas(String.valueOf(replicas));

    Builder keyBuilder = accessor.keyBuilder();

    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
  }
  return idealStates;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private void createIS(HelixDataAccessor accessor, String resourceId, String stateModelDefRef,
    RebalanceMode rebalanceMode) {
  IdealState idealState = new IdealState(resourceId);
  idealState.setRebalanceMode(rebalanceMode);
  idealState.setStateModelDefRef(stateModelDefRef);
  idealState.setNumPartitions(1);
  idealState.setReplicas("1");
  idealState.getRecord().setListField(resourceId + "_0", ImmutableList.of(PARTICIPANT));
  idealState.getRecord().setMapField(resourceId + "_0", ImmutableMap.of(PARTICIPANT, STATE));
  accessor.setProperty(accessor.keyBuilder().idealStates(resourceId), idealState);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
protected List<IdealState> setupIdealStateDeprecated(int nodes, String[] resources,
    int partitions, int replicas, IdealStateModeProperty mode) {
  List<IdealState> idealStates = new ArrayList<IdealState>();
  List<String> instances = new ArrayList<String>();
  for (int i = 0; i < nodes; i++) {
    instances.add("localhost_" + i);
  }

  for (int i = 0; i < resources.length; i++) {
    String resourceName = resources[i];
    ZNRecord record = new ZNRecord(resourceName);
    for (int p = 0; p < partitions; p++) {
      List<String> value = new ArrayList<String>();
      for (int r = 0; r < replicas; r++) {
        value.add("localhost_" + (p + r + 1) % nodes);
      }
      record.setListField(resourceName + "_" + p, value);
    }
    IdealState idealState = new IdealState(record);
    idealState.setStateModelDefRef("MasterSlave");
    idealState.setIdealStateMode(mode.toString());
    idealState.setNumPartitions(partitions);
    idealStates.add(idealState);

    // System.out.println(idealState);

    Builder keyBuilder = accessor.keyBuilder();

    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
  }
  return idealStates;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
protected List<IdealState> setupIdealState(String clusterName, int[] nodes, String[] resources,
    int partitions, int replicas) {
  ZKHelixDataAccessor accessor =
      new ZKHelixDataAccessor(clusterName, new ZkBaseDataAccessor<ZNRecord>(_gZkClient));
  Builder keyBuilder = accessor.keyBuilder();

  List<IdealState> idealStates = new ArrayList<IdealState>();
  List<String> instances = new ArrayList<String>();
  for (int i : nodes) {
    instances.add("localhost_" + i);
  }

  for (String resourceName : resources) {
    IdealState idealState = new IdealState(resourceName);
    for (int p = 0; p < partitions; p++) {
      List<String> value = new ArrayList<String>();
      for (int r = 0; r < replicas; r++) {
        int n = nodes[(p + r) % nodes.length];
        value.add("localhost_" + n);
      }
      idealState.getRecord().setListField(resourceName + "_" + p, value);
    }

    idealState.setReplicas(Integer.toString(replicas));
    idealState.setStateModelDefRef("MasterSlave");
    idealState.setRebalanceMode(RebalanceMode.SEMI_AUTO);
    idealState.setNumPartitions(partitions);
    idealStates.add(idealState);

    // System.out.println(idealState);
    accessor.setProperty(keyBuilder.idealStates(resourceName), idealState);
  }
  return idealStates;
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
private void updateIdealState() {
  HelixDataAccessor accessor = _manager.getHelixDataAccessor();
  Builder keyBuilder = accessor.keyBuilder();
  List<String> idealStates = accessor.getChildNames(keyBuilder.idealStates());
  IdealState idealState = accessor.getProperty(keyBuilder.idealStates(idealStates.get(0)));
  idealState.setNumPartitions((int)(System.currentTimeMillis()%50L));
  accessor.setProperty(keyBuilder.idealStates(idealState.getId()), idealState);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
/**
 * Add the new specified segment to the idealState of the specified table in the specified cluster.
 *
 * @param helixManager The HelixManager object to access the helix cluster.
 * @param tableName Name of the table to which the new segment is to be added.
 * @param segmentName Name of the new segment to be added
 * @param getInstancesForSegment Callable returning list of instances where the segment should be uploaded.
 */
public static void addSegmentToIdealState(HelixManager helixManager, final String tableName, final String segmentName,
    final Callable<List<String>> getInstancesForSegment) {

  Function<IdealState, IdealState> updater = new Function<IdealState, IdealState>() {
    @Override
    public IdealState apply(IdealState idealState) {
      List<String> targetInstances = null;
      try {
        targetInstances = getInstancesForSegment.call();
      } catch (Exception e) {
        LOGGER.error("Unable to get new instances for uploading segment {}, table {}", segmentName, tableName, e);
        return null;
      }

      if (targetInstances == null || targetInstances.size() == 0) {
        LOGGER.warn("No instances assigned for segment {}, table {}", segmentName, tableName);
      } else {
        for (final String instance : targetInstances) {
          idealState.setPartitionState(segmentName, instance, ONLINE);
        }
      }

      idealState.setNumPartitions(idealState.getNumPartitions() + 1);
      return idealState;
    }
  };

  updateIdealState(helixManager, tableName, updater, DEFAULT_RETRY_POLICY);
}
 

import org.apache.helix.model.IdealState; //导入方法依赖的package包/类
public static IdealState addNewRealtimeSegmentToIdealState(String segmentId, IdealState state, String instanceName) {
  state.setPartitionState(segmentId, instanceName, ONLINE);
  state.setNumPartitions(state.getNumPartitions() + 1);
  return state;
}