Java CustomModeISBuilder.build方法代码示例

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
public static IdealState buildCustomIdealStateFor(String topicName,
    int numTopicPartitions,
    PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) {

  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(numTopicPartitions).setNumReplica(1)
      .setMaxPartitionsPerNode(numTopicPartitions);

  for (int i = 0; i < numTopicPartitions; ++i) {
    synchronized (instanceToNumServingTopicPartitionMap) {
      InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll();
      customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i),
          liveInstance.getInstanceName(), "ONLINE");
      liveInstance.addTopicPartition(new TopicPartition(topicName, i));
      instanceToNumServingTopicPartitionMap.add(liveInstance);
    }
  }
  return customModeIdealStateBuilder.build();
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
public static IdealState buildCustomIdealStateFor(String topicName,
    int numTopicPartitions,
    PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) {

  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(numTopicPartitions).setNumReplica(1)
      .setMaxPartitionsPerNode(numTopicPartitions);

  for (int i = 0; i < numTopicPartitions; ++i) {
    InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll();
    if (liveInstance != null) {
      customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i),
          liveInstance.getInstanceName(), "ONLINE");
      liveInstance.addTopicPartition(new TopicPartition(topicName, i));
      instanceToNumServingTopicPartitionMap.add(liveInstance);
    }
  }
  return customModeIdealStateBuilder.build();
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
public static IdealState expandCustomRebalanceModeIdealStateFor(IdealState oldIdealState,
    String topicName, int newNumTopicPartitions,
    PriorityQueue<InstanceTopicPartitionHolder> instanceToNumServingTopicPartitionMap) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(topicName);

  customModeIdealStateBuilder
      .setStateModel(OnlineOfflineStateModel.name)
      .setNumPartitions(newNumTopicPartitions).setNumReplica(1)
      .setMaxPartitionsPerNode(newNumTopicPartitions);

  int numOldPartitions = oldIdealState.getNumPartitions();
  for (int i = 0; i < numOldPartitions; ++i) {
    String partitionName = Integer.toString(i);
    try {
      String instanceName =
          oldIdealState.getInstanceStateMap(partitionName).keySet().iterator().next();
      customModeIdealStateBuilder.assignInstanceAndState(partitionName, instanceName, "ONLINE");
    } catch (Exception e) {
      // No worker added into the cluster.
    }
  }
  for (int i = numOldPartitions; i < newNumTopicPartitions; ++i) {
    InstanceTopicPartitionHolder liveInstance = instanceToNumServingTopicPartitionMap.poll();
    customModeIdealStateBuilder.assignInstanceAndState(Integer.toString(i),
        liveInstance.getInstanceName(), "ONLINE");
    liveInstance.addTopicPartition(new TopicPartition(topicName, i));
    instanceToNumServingTopicPartitionMap.add(liveInstance);
  }
  return customModeIdealStateBuilder.build();
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
private boolean cleanUpResource(String resource, String resourceDir, FileSetInfo fileSetInfo)
    throws Exception {
  if (isResourceOfflined(resource)) {
    LOG.info("Dropping resource " + resource);
    helixAdmin.dropResource(clusterName, resource);
    zkManager.deleteViewInfo(resource);
    // Perform a recursive delete of the version directory.
    LOG.info("Cleaning up " + resourceDir);
    hdfsClient.delete(resourceDir, true);
    return true;
  } else {
    LOG.info("Offlining " + resourceDir);
    boolean enableZkCompression = configuration.getBoolean(
        Constants.ENABLE_ZK_COMPRESSION, Constants.ENABLE_ZK_COMPRESSION_DEFAULT);
    int bucketSize = TerrapinUtil.getBucketSize(
        fileSetInfo.servingInfo.numPartitions, enableZkCompression);
    if (bucketSize > 0) {
      LOG.info("Disabling resource " + resource);
      helixAdmin.enableResource(clusterName, resource, false);
    } else {
      CustomModeISBuilder offlineStateBuilder = new CustomModeISBuilder(resource);
      offlineStateBuilder.setStateModel("OnlineOffline");
      offlineStateBuilder.setNumReplica(configuration.getInt(Constants.NUM_SERVING_REPLICAS, 3));
      offlineStateBuilder.setNumPartitions(fileSetInfo.servingInfo.numPartitions);
      IdealState offlinedState = offlineStateBuilder.build();
      offlinedState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
      if (enableZkCompression) {
        TerrapinUtil.compressIdealState(offlinedState);
      }

      helixAdmin.setResourceIdealState(clusterName, resource, offlinedState);
    }
    return false;
  }
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
@Test
public void testCalculateDeviationForResource() {
  String resource = "resource";
  CustomModeISBuilder isBuilder = new CustomModeISBuilder(resource);
  isBuilder.assignInstanceAndState("0", "host1", "ONLINE");
  isBuilder.assignInstanceAndState("0", "host2", "ONLINE");
  isBuilder.assignInstanceAndState("1", "host2", "ONLINE");
  isBuilder.assignInstanceAndState("1", "host3", "ONLINE");
  isBuilder.setNumPartitions(2);
  isBuilder.setNumReplica(3);
  isBuilder.setRebalancerMode(IdealState.RebalanceMode.CUSTOMIZED);
  isBuilder.setStateModel("OnlineOffline");
  IdealState idealState = isBuilder.build();

  // Check with deviation.
  Map<String, List<InstanceConfig>> onlinePartitionMap1 = (Map)ImmutableMap.of(
      "0", ImmutableList.of(hostConfig1, hostConfig2),
      "1", ImmutableList.of(hostConfig3, hostConfig4));
  TestRoutingTableProvider routingTableProvider =
      new TestRoutingTableProvider(ImmutableMap.of(resource, onlinePartitionMap1));
  assertEquals(0.25, HdfsManager.calculateDeviationForResource(
      resource, idealState, routingTableProvider), 0.01);

  // Check perfect match.
  Map<String, List<InstanceConfig>> onlinePartitionMap2 = (Map)ImmutableMap.of(
      "0", ImmutableList.of(hostConfig1, hostConfig2),
      "1", ImmutableList.of(hostConfig2, hostConfig3));
  routingTableProvider.setOnlinePartitionMap(ImmutableMap.of(resource, onlinePartitionMap2));
  assertEquals(0.0, HdfsManager.calculateDeviationForResource(
      resource, idealState, routingTableProvider), 0.01);
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
/**
 *
 * Building an empty idealState for a given table.
 * Used when creating a new table.
 *
 * @param tableName
 * @param helixAdmin
 * @param helixClusterName
 * @return
 */
public static IdealState buildEmptyIdealStateFor(String tableName, int numCopies, HelixAdmin helixAdmin,
    String helixClusterName) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(tableName);
  final int replicas = numCopies;
  customModeIdealStateBuilder
      .setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL)
      .setNumPartitions(0).setNumReplica(replicas).setMaxPartitionsPerNode(1);
  final IdealState idealState = customModeIdealStateBuilder.build();
  idealState.setInstanceGroupTag(tableName);
  return idealState;
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
/**
 *
 * Building an empty idealState for a given table.
 * Used when creating a new table.
 *
 * @param helixAdmin
 * @param helixClusterName
 * @return
 */
public static IdealState buildEmptyIdealStateForBrokerResource(HelixAdmin helixAdmin, String helixClusterName) {
  final CustomModeISBuilder customModeIdealStateBuilder =
      new CustomModeISBuilder(CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
  customModeIdealStateBuilder
      .setStateModel(
          PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL)
      .setMaxPartitionsPerNode(Integer.MAX_VALUE).setNumReplica(Integer.MAX_VALUE)
      .setNumPartitions(Integer.MAX_VALUE);
  final IdealState idealState = customModeIdealStateBuilder.build();
  return idealState;
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
public static IdealState buildInitialKafkaHighLevelConsumerRealtimeIdealStateFor(String realtimeTableName,
    HelixAdmin helixAdmin, String helixClusterName, ZkHelixPropertyStore<ZNRecord> zkHelixPropertyStore) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(realtimeTableName);
  customModeIdealStateBuilder
      .setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL)
      .setNumPartitions(0).setNumReplica(1).setMaxPartitionsPerNode(1);
  final IdealState idealState = customModeIdealStateBuilder.build();
  idealState.setInstanceGroupTag(realtimeTableName);

  return idealState;
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
private IdealState buildWorkflowIdealState(String workflow) {
  CustomModeISBuilder IsBuilder = new CustomModeISBuilder(workflow);
  IsBuilder.setRebalancerMode(IdealState.RebalanceMode.TASK).setNumReplica(1)
      .setNumPartitions(1).setStateModel(TaskConstants.STATE_MODEL_NAME).disableExternalView();

  IdealState is = IsBuilder.build();
  is.getRecord().setListField(workflow, new ArrayList<String>());
  is.getRecord().setMapField(workflow, new HashMap<String, String>());
  is.setRebalancerClassName(WorkflowRebalancer.class.getName());

  return is;
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
/**
 *
 * Building an empty idealState for a given table.
 * Used when creating a new table.
 *
 * @param tableName resource name
 * @param numCopies is the number of replicas
 * @return
 */
public static IdealState buildEmptyIdealStateFor(String tableName, int numCopies) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(tableName);
  final int replicas = numCopies;
  customModeIdealStateBuilder
      .setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL)
      .setNumPartitions(0).setNumReplica(replicas).setMaxPartitionsPerNode(1);
  final IdealState idealState = customModeIdealStateBuilder.build();
  idealState.setInstanceGroupTag(tableName);
  return idealState;
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
public static IdealState buildEmptyKafkaConsumerRealtimeIdealStateFor(String realtimeTableName, int replicaCount) {
  final CustomModeISBuilder customModeIdealStateBuilder = new CustomModeISBuilder(realtimeTableName);
  customModeIdealStateBuilder
      .setStateModel(PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL)
      .setNumPartitions(0).setNumReplica(replicaCount).setMaxPartitionsPerNode(1);
  final IdealState idealState = customModeIdealStateBuilder.build();
  idealState.setInstanceGroupTag(realtimeTableName);

  return idealState;
}
 

import org.apache.helix.model.builder.CustomModeISBuilder; //导入方法依赖的package包/类
@Test
@PrepareForTest({ControllerUtil.class})
public void testLoadFileSet() throws Exception {
  PowerMockito.mockStatic(ControllerUtil.class);
  TerrapinLoadRequest request =
      new TerrapinLoadRequest("fileset", "/terrapin/data/fileset/12345564534", 200);
  Options requestOptions = new Options();
  requestOptions.setNumVersionsToKeep(2);
  request.setOptions(requestOptions);
  FileSetInfo fileSetInfo = new FileSetInfo(
      request.getFileSet(), "/terrapin/data/fileset/12345563434",
      request.getExpectedNumPartitions(),
      Lists.newArrayList(mock(FileSetInfo.ServingInfo.class)), new Options()
  );
  String resourceName = TerrapinUtil.hdfsDirToHelixResource(request.getHdfsDirectory());
  CustomModeISBuilder idealStateBuilder = new CustomModeISBuilder(resourceName);
  idealStateBuilder.assignInstanceAndState(resourceName + "$0", "host0", "ONLINE");
  idealStateBuilder.assignInstanceAndState(resourceName + "$1", "host1", "ONLINE");
  idealStateBuilder.setStateModel("OnlineOffline");
  idealStateBuilder.setNumReplica(2);
  idealStateBuilder.setNumPartitions(request.getExpectedNumPartitions());
  IdealState is = idealStateBuilder.build();
  is.setBucketSize(2);
  is.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
  ViewInfo viewInfo1 = mock(ViewInfo.class);
  ViewInfo viewInfo2 = mock(ViewInfo.class);

  when(configuration.getInt(eq(Constants.NUM_SERVING_REPLICAS), eq(3))).thenReturn(3);
  when(zkManager.getFileSetInfo(eq(request.getFileSet()))).thenReturn(fileSetInfo);
  when(zkManager.getViewInfo(eq(resourceName))).thenReturn(viewInfo1).thenReturn(viewInfo2);
  doNothing().when(zkManager).setFileSetInfo(eq(request.getFileSet()), any(FileSetInfo.class));
  when(viewInfo1.getNumOnlinePartitions()).thenReturn(request.getExpectedNumPartitions() / 2);
  when(viewInfo2.getNumOnlinePartitions()).thenReturn(request.getExpectedNumPartitions());
  when(helixAdmin.getResourcesInCluster(CLUSTER)).thenReturn(new ArrayList<String>());
  when(ControllerUtil.buildIdealStateForHdfsDir(any(DFSClient.class),
      anyString(), anyString(), any(PartitionerType.class), anyInt(), anyBoolean())).
          thenReturn(is);
  doNothing().when(helixAdmin).addResource(eq(CLUSTER), eq(resourceName), eq(is));
  doNothing().when(helixAdmin).addResource(eq(CLUSTER), eq(resourceName),
      eq(is.getNumPartitions()), eq("OnlineOffline"), eq("CUSTOMIZED"), eq(is.getBucketSize()));
  doNothing().when(helixAdmin).setResourceIdealState(eq(CLUSTER), eq(resourceName), eq(is));

  serviceImpl.loadFileSet(request).apply();

  ArgumentCaptor<FileSetInfo> fileSetInfoCaptor = ArgumentCaptor.forClass(FileSetInfo.class);
  verify(zkManager).setFileSetInfo(eq(request.getFileSet()), fileSetInfoCaptor.capture());
  FileSetInfo capturedInfo = fileSetInfoCaptor.getValue();
  assertEquals(request.getFileSet(), capturedInfo.fileSetName);
  assertEquals(request.getExpectedNumPartitions(), capturedInfo.servingInfo.numPartitions);
  assertEquals(Lists.newArrayList(fileSetInfo.servingInfo),
      (ArrayList)capturedInfo.oldServingInfoList);
}