Java HelixManager类代码示例

import org.apache.helix.HelixManager; //导入依赖的package包/类
/**
 * From IdealStates.
 *
 * @return InstanceToNumTopicPartitionMap
 */
public static Map<String, Set<TopicPartition>> getInstanceToTopicPartitionsMap(
    HelixManager helixManager) {
  Map<String, Set<TopicPartition>> instanceToNumTopicPartitionMap =
      new HashMap<String, Set<TopicPartition>>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    for (String partition : is.getPartitionSet()) {
      TopicPartition tpi = new TopicPartition(topic, Integer.parseInt(partition));
      for (String instance : is.getInstanceSet(partition)) {
        if (!instanceToNumTopicPartitionMap.containsKey(instance)) {
          instanceToNumTopicPartitionMap.put(instance, new HashSet<TopicPartition>());
        }
        instanceToNumTopicPartitionMap.get(instance).add(tpi);
      }
    }
  }
  return instanceToNumTopicPartitionMap;
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public static Set<TopicPartition> getUnassignedPartitions(HelixManager helixManager) {
  Set<TopicPartition> unassignedPartitions = new HashSet<TopicPartition>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    int numPartitions = is.getNumPartitions();
    for (int partition = 0; partition < numPartitions; ++partition) {
      if (is.getInstanceSet(Integer.toString(partition)).isEmpty()) {
        TopicPartition tpi = new TopicPartition(topic, partition);
        unassignedPartitions.add(tpi);
      }
    }
  }
  return unassignedPartitions;
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
/**
 * From IdealStates.
 * @param helixManager
 * @return InstanceToNumTopicPartitionMap
 */
public static Map<String, Set<TopicPartition>> getInstanceToTopicPartitionsMap(
    HelixManager helixManager) {
  Map<String, Set<TopicPartition>> instanceToNumTopicPartitionMap =
      new HashMap<String, Set<TopicPartition>>();
  HelixAdmin helixAdmin = helixManager.getClusterManagmentTool();
  String helixClusterName = helixManager.getClusterName();
  for (String topic : helixAdmin.getResourcesInCluster(helixClusterName)) {
    IdealState is = helixAdmin.getResourceIdealState(helixClusterName, topic);
    for (String partition : is.getPartitionSet()) {
      TopicPartition tpi = new TopicPartition(topic, Integer.parseInt(partition));
      for (String instance : is.getInstanceSet(partition)) {
        if (!instanceToNumTopicPartitionMap.containsKey(instance)) {
          instanceToNumTopicPartitionMap.put(instance, new HashSet<TopicPartition>());
        }
        instanceToNumTopicPartitionMap.get(instance).add(tpi);
      }
    }
  }
  return instanceToNumTopicPartitionMap;
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public GobblinHelixJobLauncher(Properties jobProps, HelixManager helixManager, Path appWorkDir) throws Exception {
  super(jobProps);

  this.helixManager = helixManager;
  this.helixTaskDriver = new TaskDriver(this.helixManager);

  this.fs = FileSystem.get(new Configuration());
  this.appWorkDir = appWorkDir;
  this.inputWorkUnitDir = new Path(appWorkDir, GobblinYarnConfigurationKeys.INPUT_WORK_UNIT_DIR_NAME);

  this.helixQueueName = this.jobContext.getJobName();
  this.jobResourceName = TaskUtil.getNamespacedJobName(this.helixQueueName, this.jobContext.getJobId());

  this.stateSerDeRunnerThreads = Integer.parseInt(jobProps.getProperty(ParallelRunner.PARALLEL_RUNNER_THREADS_KEY,
      Integer.toString(ParallelRunner.DEFAULT_PARALLEL_RUNNER_THREADS)));
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
  JobDataMap dataMap = context.getJobDetail().getJobDataMap();

  JobScheduler jobScheduler = (JobScheduler) dataMap.get(JobScheduler.JOB_SCHEDULER_KEY);
  Properties jobProps = (Properties) dataMap.get(JobScheduler.PROPERTIES_KEY);
  JobListener jobListener = (JobListener) dataMap.get(JobScheduler.JOB_LISTENER_KEY);
  HelixManager helixManager = (HelixManager) dataMap.get(GobblinHelixJobScheduler.HELIX_MANAGER_KEY);
  Path appWorkDir = (Path) dataMap.get(GobblinHelixJobScheduler.APPLICATION_WORK_DIR_KEY);

  try {
    JobLauncher jobLauncher = new GobblinHelixJobLauncher(jobProps, helixManager, appWorkDir);
    jobScheduler.runJob(jobProps, jobListener, jobLauncher);
  } catch (Throwable t) {
    throw new JobExecutionException(t);
  }
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public YarnAppSecurityManager(Config config, HelixManager helixManager, FileSystem fs) throws IOException {
  this.config = config;
  this.helixManager = helixManager;
  this.fs = fs;

  this.tokenFilePath = new Path(this.fs.getHomeDirectory(),
      config.getString(GobblinYarnConfigurationKeys.APPLICATION_NAME_KEY) + Path.SEPARATOR
          + GobblinYarnConfigurationKeys.TOKEN_FILE_NAME);
  this.fs.makeQualified(tokenFilePath);
  this.loginUser = UserGroupInformation.getLoginUser();
  this.loginIntervalInMinutes = config.getLong(GobblinYarnConfigurationKeys.LOGIN_INTERVAL_IN_MINUTES);
  this.tokenRenewIntervalInMinutes = config.getLong(GobblinYarnConfigurationKeys.TOKEN_RENEW_INTERVAL_IN_MINUTES);

  this.loginExecutor = Executors.newSingleThreadScheduledExecutor(
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("KeytabReLoginExecutor")));
  this.tokenRenewExecutor = Executors.newSingleThreadScheduledExecutor(
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("TokenRenewExecutor")));
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public static void addFakeBrokerInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant) throws Exception {
  for (int i = 0; i < numInstances; ++i) {
    final String brokerId = "Broker_localhost_" + i;
    final HelixManager helixZkManager =
        HelixManagerFactory.getZKHelixManager(helixClusterName, brokerId, InstanceType.PARTICIPANT, zkServer);
    final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine();
    final StateModelFactory<?> stateModelFactory = new EmptyBrokerOnlineOfflineStateModelFactory();
    stateMachineEngine.registerStateModelFactory(EmptyBrokerOnlineOfflineStateModelFactory.getStateModelDef(),
        stateModelFactory);
    helixZkManager.connect();
    if (isSingleTenant) {
      helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, ControllerTenantNameBuilder.getBrokerTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
    } else {
      helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, brokerId, UNTAGGED_BROKER_INSTANCE);
    }
    Thread.sleep(1000);
  }
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public static void addFakeDataInstancesToAutoJoinHelixCluster(String helixClusterName, String zkServer, int numInstances, boolean isSingleTenant) throws Exception {
  for (int i = 0; i < numInstances; ++i) {
    final String instanceId = "Server_localhost_" + i;

    final HelixManager helixZkManager =
        HelixManagerFactory.getZKHelixManager(helixClusterName, instanceId, InstanceType.PARTICIPANT, zkServer);
    final StateMachineEngine stateMachineEngine = helixZkManager.getStateMachineEngine();
    final StateModelFactory<?> stateModelFactory = new EmptySegmentOnlineOfflineStateModelFactory();
    stateMachineEngine.registerStateModelFactory(EmptySegmentOnlineOfflineStateModelFactory.getStateModelDef(),
        stateModelFactory);
    helixZkManager.connect();
    if (isSingleTenant) {
      helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId,
          TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
      helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId,
          TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
    } else {
      helixZkManager.getClusterManagmentTool().addInstanceTag(helixClusterName, instanceId, UNTAGGED_SERVER_INSTANCE);
    }
  }
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
/**
 * Remove a resource (offline/realtime table) from the Broker's ideal state.
 *
 * @param helixManager The HelixManager object for accessing helix cluster.
 * @param resourceTag Name of the resource that needs to be removed from Broker ideal state.
 */
public static void removeResourceFromBrokerIdealState(HelixManager helixManager, final String resourceTag) {
  Function<IdealState, IdealState> updater = new Function<IdealState, IdealState>() {
    @Override
    public IdealState apply(IdealState idealState) {
      if (idealState.getPartitionSet().contains(resourceTag)) {
        idealState.getPartitionSet().remove(resourceTag);
        return idealState;
      } else {
        return null;
      }
    }
  };

  // Removing partitions from ideal state
  LOGGER.info("Trying to remove resource from idealstats");
  HelixHelper.updateIdealState(helixManager, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, updater,
      DEFAULT_RETRY_POLICY);
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
/**
 * Remove the segment from the cluster.
 *
 * @param helixManager The HelixManager object to access the helix cluster.
 * @param tableName Name of the table to which the new segment is to be added.
 * @param segmentName Name of the new segment to be added
 */
public static void removeSegmentFromIdealState(HelixManager helixManager, String tableName, final String segmentName) {
  Function<IdealState, IdealState> updater = new Function<IdealState, IdealState>() {
    @Override
    public IdealState apply(IdealState idealState) {
      final Set<String> currentInstanceSet = idealState.getInstanceSet(segmentName);

      if (!currentInstanceSet.isEmpty() && idealState.getPartitionSet().contains(segmentName)) {
        idealState.getPartitionSet().remove(segmentName);
        return idealState;
      } else {
        return null;
      }
    }
  };

  updateIdealState(helixManager, tableName, updater, DEFAULT_RETRY_POLICY);
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
/**
 * Add the new specified segment to the idealState of the specified table in the specified cluster.
 *
 * @param helixManager The HelixManager object to access the helix cluster.
 * @param tableName Name of the table to which the new segment is to be added.
 * @param segmentName Name of the new segment to be added
 * @param getInstancesForSegment Callable returning list of instances where the segment should be uploaded.
 */
public static void addSegmentToIdealState(HelixManager helixManager, String tableName, final String segmentName,
    final Callable<List<String>> getInstancesForSegment) {

  Function<IdealState, IdealState> updater = new Function<IdealState, IdealState>() {
    @Override
    public IdealState apply(IdealState idealState) {
      List<String> targetInstances = null;
      try {
        targetInstances = getInstancesForSegment.call();
      } catch (Exception e) {
        LOGGER.error("Unable to get new instances for segment uploading.");
        return null;
      }

      for (final String instance : targetInstances) {
        idealState.setPartitionState(segmentName, instance, ONLINE);
      }

      idealState.setNumPartitions(idealState.getNumPartitions() + 1);
      return idealState;
    }
  };

  updateIdealState(helixManager, tableName, updater, DEFAULT_RETRY_POLICY);
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public TaskStateModelFactory(HelixManager manager, Map<String, TaskFactory> taskFactoryRegistry,
    ScheduledExecutorService taskExecutor) {
  _manager = manager;
  _taskFactoryRegistry = taskFactoryRegistry;
  _taskExecutor = taskExecutor;
  _timerTaskExecutor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
    @Override public Thread newThread(Runnable r) {
      return new Thread(r, "TaskStateModelFactory-timeTask_thread");
    }
  });
  if (_taskExecutor instanceof ThreadPoolExecutor) {
    try {
      _monitor = new ThreadPoolExecutorMonitor(TaskConstants.STATE_MODEL_NAME,
          (ThreadPoolExecutor) _taskExecutor);
    } catch (JMException e) {
      LOG.warn("Error in creating ThreadPoolExecutorMonitor for TaskStateModelFactory.");
    }
  }
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
protected PinotLLCRealtimeSegmentManager(HelixAdmin helixAdmin, String clusterName, HelixManager helixManager,
    ZkHelixPropertyStore propertyStore, PinotHelixResourceManager helixResourceManager, ControllerConf controllerConf,
    ControllerMetrics controllerMetrics) {
  _helixAdmin = helixAdmin;
  _helixManager = helixManager;
  _propertyStore = propertyStore;
  _helixResourceManager = helixResourceManager;
  _clusterName = clusterName;
  _controllerConf = controllerConf;
  _controllerMetrics = controllerMetrics;
  _idealstateUpdateLocks = new Lock[NUM_LOCKS];
  for (int i = 0; i < NUM_LOCKS; i++) {
    _idealstateUpdateLocks[i] = new ReentrantLock();
  }
  _tableConfigCache = new TableConfigCache(_propertyStore);
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
public YarnAppSecurityManager(Config config, HelixManager helixManager, FileSystem fs, Path tokenFilePath)
    throws IOException {
  this.config = config;
  this.helixManager = helixManager;
  this.fs = fs;

  this.tokenFilePath = tokenFilePath;
  this.fs.makeQualified(tokenFilePath);
  this.loginUser = UserGroupInformation.getLoginUser();
  this.loginIntervalInMinutes = config.getLong(GobblinYarnConfigurationKeys.LOGIN_INTERVAL_IN_MINUTES);
  this.tokenRenewIntervalInMinutes = config.getLong(GobblinYarnConfigurationKeys.TOKEN_RENEW_INTERVAL_IN_MINUTES);

  this.loginExecutor = Executors.newSingleThreadScheduledExecutor(
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("KeytabReLoginExecutor")));
  this.tokenRenewExecutor = Executors.newSingleThreadScheduledExecutor(
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("TokenRenewExecutor")));
}
 

import org.apache.helix.HelixManager; //导入依赖的package包/类
@Override
public void executeImpl(JobExecutionContext context) throws JobExecutionException {
  JobDataMap dataMap = context.getJobDetail().getJobDataMap();
  ConcurrentHashMap runningMap = (ConcurrentHashMap)dataMap.get(GobblinHelixJobScheduler.JOB_RUNNING_MAP);
  final JobScheduler jobScheduler = (JobScheduler) dataMap.get(JobScheduler.JOB_SCHEDULER_KEY);
  // the properties may get mutated during job execution and the scheduler reuses it for the next round of scheduling,
  // so clone it
  final Properties jobProps = (Properties)((Properties) dataMap.get(JobScheduler.PROPERTIES_KEY)).clone();
  final JobListener jobListener = (JobListener) dataMap.get(JobScheduler.JOB_LISTENER_KEY);
  HelixManager helixManager = (HelixManager) dataMap.get(GobblinHelixJobScheduler.HELIX_MANAGER_KEY);
  Path appWorkDir = (Path) dataMap.get(GobblinHelixJobScheduler.APPLICATION_WORK_DIR_KEY);
  @SuppressWarnings("unchecked")
  List<? extends Tag<?>> eventMetadata = (List<? extends Tag<?>>) dataMap.get(GobblinHelixJobScheduler.METADATA_TAGS);

  try {
    final JobLauncher jobLauncher = new GobblinHelixJobLauncher(jobProps, helixManager, appWorkDir, eventMetadata, runningMap);
    if (Boolean.valueOf(jobProps.getProperty(GobblinClusterConfigurationKeys.JOB_EXECUTE_IN_SCHEDULING_THREAD,
            Boolean.toString(GobblinClusterConfigurationKeys.JOB_EXECUTE_IN_SCHEDULING_THREAD_DEFAULT)))) {
      jobScheduler.runJob(jobProps, jobListener, jobLauncher);
    } else {
      cancellable = jobScheduler.scheduleJobImmediately(jobProps, jobListener, jobLauncher);
    }
  } catch (Throwable t) {
    throw new JobExecutionException(t);
  }
}