Java LeafQueue类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent,
    NodeLabel nodeLabel) {
  CSQueue parentQueue = parent;
  CapacitySchedulerQueueInfoList queuesInfo =
      new CapacitySchedulerQueueInfoList();
  for (CSQueue queue : parentQueue.getChildQueues()) {
    CapacitySchedulerQueueInfo info;
    if (queue instanceof LeafQueue) {
      info =
          new CapacitySchedulerLeafQueueInfo((LeafQueue) queue,
              nodeLabel.getLabelName());
    } else {
      info = new CapacitySchedulerQueueInfo(queue, nodeLabel.getLabelName());
      info.queues = getQueues(queue, nodeLabel);
    }
    queuesInfo.addToQueueInfoList(info);
  }
  return queuesInfo;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
void printString(CSQueue nq, String indent) {
  if (nq instanceof ParentQueue) {
    System.out.println(indent + nq.getQueueName()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (CSQueue q : ((ParentQueue)nq).getChildQueues()) {
      printString(q, indent + "  ");
    }
  } else {
    System.out.println(indent + nq.getQueueName()
        + " pen:" + ((LeafQueue) nq).getTotalResourcePending()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (FiCaSchedulerApp a : ((LeafQueue)nq).getApplications()) {
      System.out.println(indent + "  " + a.getApplicationId());
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
CapacitySchedulerLeafQueueInfo(LeafQueue q) {
  super(q);
  numActiveApplications = q.getNumActiveApplications();
  numPendingApplications = q.getNumPendingApplications();
  numContainers = q.getNumContainers();
  maxApplications = q.getMaxApplications();
  maxApplicationsPerUser = q.getMaxApplicationsPerUser();
  userLimit = q.getUserLimit();
  users = new UsersInfo(q.getUsers());
  userLimitFactor = q.getUserLimitFactor();
  AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
  usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
  userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit());
  preemptionDisabled = q.getPreemptionDisabled();
  orderingPolicyInfo = q.getOrderingPolicy().getInfo();
  defaultNodeLabelExpression = q.getDefaultNodeLabelExpression();
  defaultPriority = q.getDefaultApplicationPriority().getPriority();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) {
  CSQueue parentQueue = parent;
  CapacitySchedulerQueueInfoList queuesInfo =
      new CapacitySchedulerQueueInfoList();
  for (CSQueue queue : parentQueue.getChildQueues()) {
    CapacitySchedulerQueueInfo info;
    if (queue instanceof LeafQueue) {
      info =
          new CapacitySchedulerLeafQueueInfo((LeafQueue) queue);
    } else {
      info = new CapacitySchedulerQueueInfo(queue);
      info.queues = getQueues(queue);
    }
    queuesInfo.addToQueueInfoList(info);
  }
  return queuesInfo;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
protected void getPendingAppDiagnosticMessage(
    StringBuilder diagnosticMessage) {
  LeafQueue queue = getCSLeafQueue();
  diagnosticMessage.append(" Details : AM Partition = ");
  diagnosticMessage.append(appAMNodePartitionName.isEmpty()
      ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
  diagnosticMessage.append("; ");
  diagnosticMessage.append("AM Resource Request = ");
  diagnosticMessage.append(getAMResource(appAMNodePartitionName));
  diagnosticMessage.append("; ");
  diagnosticMessage.append("Queue Resource Limit for AM = ");
  diagnosticMessage
      .append(queue.getAMResourceLimitPerPartition(appAMNodePartitionName));
  diagnosticMessage.append("; ");
  diagnosticMessage.append("User AM Resource Limit of the queue = ");
  diagnosticMessage.append(
      queue.getUserAMResourceLimitPerPartition(appAMNodePartitionName));
  diagnosticMessage.append("; ");
  diagnosticMessage.append("Queue AM Resource Usage = ");
  diagnosticMessage.append(
      queue.getQueueResourceUsage().getAMUsed(appAMNodePartitionName));
  diagnosticMessage.append("; ");
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
protected void getActivedAppDiagnosticMessage(
    StringBuilder diagnosticMessage) {
  LeafQueue queue = getCSLeafQueue();
  QueueCapacities queueCapacities = queue.getQueueCapacities();
  diagnosticMessage.append(" Details : AM Partition = ");
  diagnosticMessage.append(appAMNodePartitionName.isEmpty()
      ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
  diagnosticMessage.append(" ; ");
  diagnosticMessage.append("Partition Resource = ");
  diagnosticMessage.append(rmContext.getNodeLabelManager()
      .getResourceByLabel(appAMNodePartitionName, Resources.none()));
  diagnosticMessage.append(" ; ");
  diagnosticMessage.append("Queue's Absolute capacity = ");
  diagnosticMessage.append(
      queueCapacities.getAbsoluteCapacity(appAMNodePartitionName) * 100);
  diagnosticMessage.append(" % ; ");
  diagnosticMessage.append("Queue's Absolute used capacity = ");
  diagnosticMessage.append(
      queueCapacities.getAbsoluteUsedCapacity(appAMNodePartitionName) * 100);
  diagnosticMessage.append(" % ; ");
  diagnosticMessage.append("Queue's Absolute max capacity = ");
  diagnosticMessage.append(
      queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
          * 100);
  diagnosticMessage.append(" % ; ");
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
void printString(CSQueue nq, String indent) {
  if (nq instanceof ParentQueue) {
    System.out.println(indent + nq.getQueueName()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (CSQueue q : ((ParentQueue)nq).getChildQueues()) {
      printString(q, indent + "  ");
    }
  } else {
    System.out.println(indent + nq.getQueueName()
        + " pen:"
            + ((LeafQueue) nq).getTotalPendingResourcesConsideringUserLimit(
                                      isA(Resource.class), isA(String.class))
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (FiCaSchedulerApp a : ((LeafQueue)nq).getApplications()) {
      System.out.println(indent + "  " + a.getApplicationId());
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm2, app, clusterResource, queueResource, usedResource,
    numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources = Resources.subtract(queueResource, usedResource);
  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  asserteMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  asserteMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getTotalConsumedResources());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
private Resource calculateUsedAMResourcesPerQueue(String partition,
    LeafQueue leafQueue, Map<String, Resource> perUserAMUsed) {
  Collection<FiCaSchedulerApp> runningApps = leafQueue.getApplications();
  Resource amUsed = Resources.createResource(0, 0);

  for (FiCaSchedulerApp app : runningApps) {
    Resource userAMResource = perUserAMUsed.get(app.getUser());
    if (null == userAMResource) {
      userAMResource = Resources.createResource(0, 0);
      perUserAMUsed.put(app.getUser(), userAMResource);
    }

    Resources.addTo(userAMResource, app.getAMResource(partition));
    Resources.addTo(amUsed, app.getAMResource(partition));
  }
  return amUsed;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
CapacitySchedulerLeafQueueInfo(LeafQueue q, String nodeLabel) {
  super(q, nodeLabel);
  numActiveApplications = q.getNumActiveApplications();
  numPendingApplications = q.getNumPendingApplications();
  numContainers = q.getNumContainers();
  maxApplications = q.getMaxApplications();
  maxApplicationsPerUser = q.getMaxApplicationsPerUser();
  userLimit = q.getUserLimit();
  users = new UsersInfo(q.getUsers());
  userLimitFactor = q.getUserLimitFactor();
  AMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
  usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed());
  userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit());
  preemptionDisabled = q.getPreemptionDisabled();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
LeafQueue mockLeafQueue(ParentQueue p, float tot, int i, int[] abs, 
    int[] used, int[] pending, int[] reserved, int[] apps, int[] gran) {
  LeafQueue lq = mock(LeafQueue.class);
  List<ApplicationAttemptId> appAttemptIdList = 
      new ArrayList<ApplicationAttemptId>();
  when(lq.getTotalResourcePending()).thenReturn(
      Resource.newInstance(pending[i], 0, 0));
  // consider moving where CapacityScheduler::comparator accessible
  NavigableSet<FiCaSchedulerApp> qApps = new TreeSet<FiCaSchedulerApp>(
    new Comparator<FiCaSchedulerApp>() {
      @Override
      public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
        return a1.getApplicationAttemptId()
            .compareTo(a2.getApplicationAttemptId());
      }
    });
  // applications are added in global L->R order in queues
  if (apps[i] != 0) {
    int aUsed    = used[i] / apps[i];
    int aPending = pending[i] / apps[i];
    int aReserve = reserved[i] / apps[i];
    for (int a = 0; a < apps[i]; ++a) {
      FiCaSchedulerApp mockFiCaApp =
          mockApp(i, appAlloc, aUsed, aPending, aReserve, gran[i]);
      qApps.add(mockFiCaApp);
      ++appAlloc;
      appAttemptIdList.add(mockFiCaApp.getApplicationAttemptId());
    }
    when(mCS.getAppsInQueue("queue" + (char)('A' + i - 1)))
            .thenReturn(appAttemptIdList);
  }
  when(lq.getApplications()).thenReturn(qApps);
  if(setAMResourcePercent != 0.0f){
    when(lq.getMaxAMResourcePerQueuePercent()).thenReturn(setAMResourcePercent);
  }
  p.getChildQueues().add(lq);
  return lq;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; //导入依赖的package包/类
private FiCaSchedulerApp getApp(String queueName, int appId) {
  for (FiCaSchedulerApp app : ((LeafQueue) cs.getQueue(queueName))
      .getApplications()) {
    if (app.getApplicationId().getId() == appId) {
      return app;
    }
  }
  return null;
}