Java SchedulerApplication类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
@VisibleForTesting
public synchronized void addApplication(ApplicationId applicationId,
    String queue, String user, boolean isAppRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      new SchedulerApplication<FiCaSchedulerApp>(DEFAULT_QUEUE, user);
  applications.put(applicationId, application);
  metrics.submitApp(user);
  LOG.info("Accepted application " + applicationId + " from user: " + user
      + ", currently num of applications: " + applications.size());
  if (isAppRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationId + " is recovering. Skip notifying APP_ACCEPTED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED));
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationId);
  if (application == null){
    // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps,
    // ignore it.
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();
  if (!(queue instanceof LeafQueue)) {
    LOG.error("Cannot finish application " + "from non-leaf queue: "
        + queue.getQueueName());
  } else {
    queue.finishApplication(applicationId, application.getUser());
  }
  application.stop(finalState);
  applications.remove(applicationId);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private void checkAppQueue(MockRM resourceManager, String user,
    String submissionQueue, String expected)
    throws Exception {
  RMApp app = resourceManager.submitApp(200, "name", user,
      new HashMap<ApplicationAccessType, String>(), false, submissionQueue, -1,
      null, "MAPREDUCE", false);
  RMAppState expectedState = expected.isEmpty() ? RMAppState.FAILED
      : RMAppState.ACCEPTED;
  resourceManager.waitForState(app.getApplicationId(), expectedState);
  // get scheduler app
  CapacityScheduler cs = (CapacityScheduler)
      resourceManager.getResourceScheduler();
  SchedulerApplication schedulerApp =
      cs.getSchedulerApplications().get(app.getApplicationId());
  String queue = "";
  if (schedulerApp != null) {
    queue = schedulerApp.getQueue().getQueueName();
  }
  Assert.assertTrue("expected " + expected + " actual " + queue,
      expected.equals(queue));
  Assert.assertEquals(expected, app.getQueue());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm2, app, clusterResource, queueResource, usedResource,
    numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources = Resources.subtract(queueResource, usedResource);
  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  asserteMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  asserteMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getTotalConsumedResources());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private void checkFifoQueue(SchedulerApplication schedulerApp,
    Resource usedResources, Resource availableResources) throws Exception {
  FifoScheduler scheduler = (FifoScheduler) rm2.getResourceScheduler();
  // ************ check cluster used Resources ********
  assertEquals(usedResources, scheduler.getUsedResource());

  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt =
      schedulerApp.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************ check queue metrics ****************
  QueueMetrics queueMetrics = scheduler.getRootQueueMetrics();
  asserteMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResources.getMemory(),
    usedResources.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
public synchronized void unreserveResource(
    SchedulerApplication application) {
  
  // adding NP checks as this can now be called for preemption
  if (reservedContainer != null
      && reservedContainer.getContainer() != null
      && reservedContainer.getContainer().getId() != null
      && reservedContainer.getContainer().getId().getApplicationAttemptId() != null) {

    // Cannot unreserve for wrong application...
    ApplicationAttemptId reservedApplication =
        reservedContainer.getContainer().getId().getApplicationAttemptId();
    if (!reservedApplication.equals(
        application.getApplicationAttemptId())) {
      throw new IllegalStateException("Trying to unreserve " +
          " for application " + application.getApplicationAttemptId() +
          " when currently reserved " +
          " for application " + reservedApplication.getApplicationId() +
          " on node " + this);
    }
  }
  reservedContainer = null;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
/**
 * Wait until an app is removed from scheduler.
 * @param appId the id of an app
 * @param timeoutMsecs the length of timeout in milliseconds
 * @throws InterruptedException
 *         if interrupted while waiting for app removed
 */
public void waitForAppRemovedFromScheduler(ApplicationId appId,
    long timeoutMsecs) throws InterruptedException {
  int timeWaiting = 0;

  Map<ApplicationId, SchedulerApplication> apps  =
      ((AbstractYarnScheduler) getResourceScheduler())
          .getSchedulerApplications();
  while (apps.containsKey(appId)) {
    if (timeWaiting >= timeoutMsecs) {
      break;
    }
    LOG.info("wait for app removed, " + appId);
    Thread.sleep(WAIT_MS_PER_LOOP);
    timeWaiting += WAIT_MS_PER_LOOP;
  }
  Assert.assertTrue("app is not removed from scheduler (timeout).",
      !apps.containsKey(appId));
  LOG.info("app is removed from scheduler, " + appId);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private synchronized void
    addApplicationAttempt(ApplicationAttemptId appAttemptId,
        boolean transferStateFromPreviousAttempt) {
  SchedulerApplication application =
      applications.get(appAttemptId.getApplicationId());
  String user = application.getUser();
  // TODO: Fix store
  FiCaSchedulerApp schedulerApp =
      new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE,
        activeUsersManager, this.rmContext);

  if (transferStateFromPreviousAttempt) {
    schedulerApp.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(schedulerApp);

  metrics.submitAppAttempt(user);
  LOG.info("Added Application Attempt " + appAttemptId
      + " to scheduler from user " + application.getUser());
  rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(appAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt) {
  SchedulerApplication application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  rmContext.getDispatcher().getEventHandler() .handle(
      new RMAppAttemptEvent(applicationAttemptId,
        RMAppAttemptEventType.ATTEMPT_ADDED));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入依赖的package包/类
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication application = applications.get(applicationId);
  if (application == null){
    // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps,
    // ignore it.
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();
  if (!(queue instanceof LeafQueue)) {
    LOG.error("Cannot finish application " + "from non-leaf queue: "
        + queue.getQueueName());
  } else {
    queue.finishApplication(applicationId, application.getUser());
  }
  application.stop(finalState);
  applications.remove(applicationId);
}