Java ContainerPreemptEventType类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; //导入依赖的package包/类
protected void createPolicyMonitors() {
  if (scheduler instanceof PreemptableResourceScheduler
      && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
    LOG.info("Loading policy monitors");
    List<SchedulingEditPolicy> policies = conf.getInstances(
        YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
        SchedulingEditPolicy.class);
    if (policies.size() > 0) {
      rmDispatcher.register(ContainerPreemptEventType.class,
          new RMContainerPreemptEventDispatcher(
              (PreemptableResourceScheduler) scheduler));
      for (SchedulingEditPolicy policy : policies) {
        LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
        // periodically check whether we need to take action to guarantee
        // constraints
        SchedulingMonitor mon = new SchedulingMonitor(rmContext, policy);
        addService(mon);
      }
    } else {
      LOG.warn("Policy monitors configured (" +
          YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
          ") but none specified (" +
          YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; //导入依赖的package包/类
protected void createPolicyMonitors() {
  if (scheduler instanceof PreemptableResourceScheduler
      && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
        YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
    LOG.info("Loading policy monitors");
    List<SchedulingEditPolicy> policies = conf.getInstances(
            YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
            SchedulingEditPolicy.class);
    if (policies.size() > 0) {
      this.rmDispatcher.register(ContainerPreemptEventType.class,
        new RMContainerPreemptEventDispatcher(
          (PreemptableResourceScheduler) scheduler));
      for (SchedulingEditPolicy policy : policies) {
        LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
        policy.init(conf, this.rmContext.getDispatcher().getEventHandler(),
            (PreemptableResourceScheduler) scheduler);
        // periodically check whether we need to take action to guarantee
        // constraints
        SchedulingMonitor mon = new SchedulingMonitor(policy);
        addService(mon);

      }
    } else {
      LOG.warn("Policy monitors configured (" +
          YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
          ") but none specified (" +
          YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; //导入依赖的package包/类
protected void createPolicyMonitors() {
  if (scheduler instanceof PreemptableResourceScheduler
      && conf.getBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ENABLE_MONITORS)) {
    LOG.info("Loading policy monitors");
    List<SchedulingEditPolicy> policies = conf.getInstances(
        YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
        SchedulingEditPolicy.class);
    if (policies.size() > 0) {
      rmDispatcher.register(ContainerPreemptEventType.class,
          new RMContainerPreemptEventDispatcher(
              (PreemptableResourceScheduler) scheduler));
      for (SchedulingEditPolicy policy : policies) {
        LOG.info("LOADING SchedulingEditPolicy:" + policy.getPolicyName());
        policy.init(conf, rmContext.getDispatcher().getEventHandler(),
            (PreemptableResourceScheduler) scheduler);
        // periodically check whether we need to take action to guarantee
        // constraints
        SchedulingMonitor mon = new SchedulingMonitor(policy);
        addService(mon);
      }
    } else {
      LOG.warn("Policy monitors configured (" +
          YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS +
          ") but none specified (" +
          YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES + ")");
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; //导入依赖的package包/类
/**
 * This method selects and tracks containers to be preempted. If a container
 * is in the target list for more than maxWaitTime it is killed.
 *
 * @param root the root of the CapacityScheduler queue hierarchy
 * @param clusterResources the total amount of resources in the cluster
 */
private void containerBasedPreemptOrKill(CSQueue root,
    Resource clusterResources) {

  // extract a summary of the queues from scheduler
  TempQueue tRoot;
  synchronized (scheduler) {
    tRoot = cloneQueues(root, clusterResources);
  }

  // compute the ideal distribution of resources among queues
  // updates cloned queues state accordingly
  tRoot.idealAssigned = tRoot.guaranteed;
  Resource totalPreemptionAllowed = Resources.multiply(clusterResources,
      percentageClusterPreemptionAllowed);
  List<TempQueue> queues =
    recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed);

  // based on ideal allocation select containers to be preempted from each
  // queue and each application
  Map<ApplicationAttemptId,Set<RMContainer>> toPreempt =
      getContainersToPreempt(queues, clusterResources);

  if (LOG.isDebugEnabled()) {
    logToCSV(queues);
  }

  // if we are in observeOnly mode return before any action is taken
  if (observeOnly) {
    return;
  }

  // preempt (or kill) the selected containers
  for (Map.Entry<ApplicationAttemptId,Set<RMContainer>> e
       : toPreempt.entrySet()) {
    for (RMContainer container : e.getValue()) {
      // if we tried to preempt this for more than maxWaitTime
      if (preempted.get(container) != null &&
          preempted.get(container) + maxWaitTime < clock.getTime()) {
        // kill it
        dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
              ContainerPreemptEventType.KILL_CONTAINER));
        preempted.remove(container);
      } else {
        //otherwise just send preemption events
        dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
              ContainerPreemptEventType.PREEMPT_CONTAINER));
        if (preempted.get(container) == null) {
          preempted.put(container, clock.getTime());
        }
      }
    }
  }

  // Keep the preempted list clean
  for (Iterator<RMContainer> i = preempted.keySet().iterator(); i.hasNext();){
    RMContainer id = i.next();
    // garbage collect containers that are irrelevant for preemption
    if (preempted.get(id) + 2 * maxWaitTime < clock.getTime()) {
      i.remove();
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; //导入依赖的package包/类
IsPreemptionRequestFor(ApplicationAttemptId appAttId,
    ContainerPreemptEventType type) {
  this.appAttId = appAttId;
  this.type = type;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerPreemptEventType; //导入依赖的package包/类
/**
 * This method selects and tracks containers to be preempted. If a container
 * is in the target list for more than maxWaitTime it is killed.
 *
 * @param root the root of the CapacityScheduler queue hierarchy
 * @param clusterResources the total amount of resources in the cluster
 */
private void containerBasedPreemptOrKill(CSQueue root,
    Resource clusterResources) {

  // extract a summary of the queues from scheduler
  TempQueue tRoot;
  synchronized (scheduler) {
    tRoot = cloneQueues(root, clusterResources);
  }

  // compute the ideal distribution of resources among queues
  // updates cloned queues state accordingly
  tRoot.idealAssigned = tRoot.guaranteed;
  Resource totalPreemptionAllowed = Resources.multiply(clusterResources,
      percentageClusterPreemptionAllowed);
  List<TempQueue> queues =
    recursivelyComputeIdealAssignment(tRoot, totalPreemptionAllowed);

  // based on ideal allocation select containers to be preempted from each
  // queue and each application
  Map<ApplicationAttemptId,Set<RMContainer>> toPreempt =
      getContainersToPreempt(queues, clusterResources);

  logToCSV(queues);

  // if we are in observeOnly mode return before any action is taken
  if (observeOnly) {
    return;
  }

  // preempt (or kill) the selected containers
  for (Map.Entry<ApplicationAttemptId,Set<RMContainer>> e
       : toPreempt.entrySet()) {
    for (RMContainer container : e.getValue()) {
      // if we tried to preempt this for more than maxWaitTime
      if (preempted.get(container) != null &&
          preempted.get(container) + maxWaitTime < clock.getTime()) {
        // kill it
        dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
              ContainerPreemptEventType.KILL_CONTAINER));
        preempted.remove(container);
      } else {
        //otherwise just send preemption events
        dispatcher.handle(new ContainerPreemptEvent(e.getKey(), container,
              ContainerPreemptEventType.PREEMPT_CONTAINER));
        if (preempted.get(container) == null) {
          preempted.put(container, clock.getTime());
        }
      }
    }
  }

  // Keep the preempted list clean
  for (Iterator<RMContainer> i = preempted.keySet().iterator(); i.hasNext();){
    RMContainer id = i.next();
    // garbage collect containers that are irrelevant for preemption
    if (preempted.get(id) + 2 * maxWaitTime < clock.getTime()) {
      i.remove();
    }
  }
}