Java SchedulerApplication.getQueue方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationId);
  if (application == null){
    // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps,
    // ignore it.
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();
  if (!(queue instanceof LeafQueue)) {
    LOG.error("Cannot finish application " + "from non-leaf queue: "
        + queue.getQueueName());
  } else {
    queue.finishApplication(applicationId, application.getUser());
  }
  application.stop(finalState);
  applications.remove(applicationId);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm2, app, clusterResource, queueResource, usedResource,
    numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources = Resources.subtract(queueResource, usedResource);
  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  asserteMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  asserteMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getTotalConsumedResources());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt) {
  SchedulerApplication application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  rmContext.getDispatcher().getEventHandler() .handle(
      new RMAppAttemptEvent(applicationAttemptId,
        RMAppAttemptEventType.ATTEMPT_ADDED));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication application = applications.get(applicationId);
  if (application == null){
    // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps,
    // ignore it.
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();
  if (!(queue instanceof LeafQueue)) {
    LOG.error("Cannot finish application " + "from non-leaf queue: "
        + queue.getQueueName());
  } else {
    queue.finishApplication(applicationId, application.getUser());
  }
  application.stop(finalState);
  applications.remove(applicationId);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
@Override
public synchronized String moveApplication(ApplicationId appId,
    String queueName) throws YarnException {
  SchedulerApplication<FSAppAttempt> app = applications.get(appId);
  if (app == null) {
    throw new YarnException("App to be moved " + appId + " not found.");
  }
  FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt();
  // To serialize with FairScheduler#allocate, synchronize on app attempt
  synchronized (attempt) {
    FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
    String destQueueName = handleMoveToPlanQueue(queueName);
    FSLeafQueue targetQueue = queueMgr.getLeafQueue(destQueueName, false);
    if (targetQueue == null) {
      throw new YarnException("Target queue " + queueName
          + " not found or is not a leaf queue.");
    }
    if (targetQueue == oldQueue) {
      return oldQueue.getQueueName();
    }
    
    if (oldQueue.isRunnableApp(attempt)) {
      verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue);
    }
    
    executeMove(app, attempt, oldQueue, targetQueue);
    return targetQueue.getQueueName();
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
@Override
public synchronized String moveApplication(ApplicationId appId,
    String queueName) throws YarnException {
  SchedulerApplication<FSAppAttempt> app = applications.get(appId);
  if (app == null) {
    throw new YarnException("App to be moved " + appId + " not found.");
  }
  FSAppAttempt attempt = (FSAppAttempt) app.getCurrentAppAttempt();
  // To serialize with FairScheduler#allocate, synchronize on app attempt
  synchronized (attempt) {
    FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
    FSLeafQueue targetQueue = queueMgr.getLeafQueue(queueName, false);
    if (targetQueue == null) {
      throw new YarnException("Target queue " + queueName
          + " not found or is not a leaf queue.");
    }
    if (targetQueue == oldQueue) {
      return oldQueue.getQueueName();
    }
    
    if (oldQueue.isRunnableApp(attempt)) {
      verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue);
    }
    
    executeMove(app, attempt, oldQueue, targetQueue);
    return targetQueue.getQueueName();
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(),
      availableResources.getVirtualCores(), usedResource.getMemorySize(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(),
      availableResources.getVirtualCores(), usedResource.getMemorySize(),
      usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
/**
 * Add a new application attempt to the scheduler.
 */
protected synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt) {
  SchedulerApplication application =
      applications.get(applicationAttemptId.getApplicationId());
  String user = application.getUser();
  FSLeafQueue queue = (FSLeafQueue) application.getQueue();

  FSSchedulerApp attempt =
      new FSSchedulerApp(applicationAttemptId, user,
          queue, new ActiveUsersManager(getRootQueueMetrics()),
          rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, user);
  queue.addApp(attempt, runnable);
  if (runnable) {
    maxRunningEnforcer.trackRunnableApp(attempt);
  } else {
    maxRunningEnforcer.trackNonRunnableApp(attempt);
  }
  
  queue.getMetrics().submitAppAttempt(user);

  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user: " + user);
  rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
@Override
public synchronized String moveApplication(ApplicationId appId,
    String queueName) throws YarnException {
  SchedulerApplication app = applications.get(appId);
  if (app == null) {
    throw new YarnException("App to be moved " + appId + " not found.");
  }
  FSSchedulerApp attempt = (FSSchedulerApp) app.getCurrentAppAttempt();
  // To serialize with FairScheduler#allocate, synchronize on app attempt
  synchronized (attempt) {
    FSLeafQueue oldQueue = (FSLeafQueue) app.getQueue();
    FSLeafQueue targetQueue = queueMgr.getLeafQueue(queueName, false);
    if (targetQueue == null) {
      throw new YarnException("Target queue " + queueName
          + " not found or is not a leaf queue.");
    }
    if (targetQueue == oldQueue) {
      return oldQueue.getQueueName();
    }
    
    if (oldQueue.getRunnableAppSchedulables().contains(
        attempt.getAppSchedulable())) {
      verifyMoveDoesNotViolateConstraints(attempt, oldQueue, targetQueue);
    }
    
    executeMove(app, attempt, oldQueue, targetQueue);
    return targetQueue.getQueueName();
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
/**
 * Add a new application attempt to the scheduler.
 */
protected synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FSAppAttempt> application =
      applications.get(applicationAttemptId.getApplicationId());
  String user = application.getUser();
  FSLeafQueue queue = (FSLeafQueue) application.getQueue();

  FSAppAttempt attempt =
      new FSAppAttempt(this, applicationAttemptId, user,
          queue, new ActiveUsersManager(getRootQueueMetrics()),
          rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
        .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, user);
  queue.addApp(attempt, runnable);
  if (runnable) {
    maxRunningEnforcer.trackRunnableApp(attempt);
  } else {
    maxRunningEnforcer.trackNonRunnableApp(attempt);
  }
  
  queue.getMetrics().submitAppAttempt(user);

  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user: " + user);

  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication; //导入方法依赖的package包/类
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  if (application == null) {
    LOG.warn("Application " + applicationAttemptId.getApplicationId() +
        " cannot be found in scheduler.");
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt = new FiCaSchedulerApp(applicationAttemptId,
      application.getUser(), queue, queue.getActiveUsersManager(), rmContext,
      application.getPriority());
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(
        application.getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  // Update attempt priority to the latest to avoid race condition i.e
  // SchedulerApplicationAttempt is created with old priority but it is not
  // set to SchedulerApplication#setCurrentAppAttempt.
  // Scenario would occur is
  // 1. SchdulerApplicationAttempt is created with old priority.
  // 2. updateApplicationPriority() updates SchedulerApplication. Since
  // currentAttempt is null, it just return.
  // 3. ScheduelerApplcationAttempt is set in
  // SchedulerApplication#setCurrentAppAttempt.
  attempt.setPriority(application.getPriority());

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}