Java ResourceManager.getResourceScheduler方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
  this.maximumClusterPriority =
      rs.getMaxClusterLevelAppPriority().getPriority();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
@Inject QueuesBlock(ResourceManager rm, CSQInfo info) {
  cs = (CapacityScheduler) rm.getResourceScheduler();
  csqinfo = info;
  RMNodeLabelsManager nodeLabelManager =
      rm.getRMContext().getNodeLabelManager();
  nodeLabelsInfo = nodeLabelManager.pullRMNodeLabelsInfo();      
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public FifoSchedulerInfo(final ResourceManager rm) {

    RMContext rmContext = rm.getRMContext();

    FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();
    qName = fs.getQueueInfo("", false, false).getQueueName();
    QueueInfo qInfo = fs.getQueueInfo(qName, true, true);

    this.usedCapacity = qInfo.getCurrentCapacity();
    this.capacity = qInfo.getCapacity();
    this.minQueueMemoryCapacity = fs.getMinimumResourceCapability().getMemory();
    this.maxQueueMemoryCapacity = fs.getMaximumResourceCapability().getMemory();
    this.qstate = qInfo.getQueueState();

    this.numNodes = rmContext.getRMNodes().size();
    this.usedNodeCapacity = 0;
    this.availNodeCapacity = 0;
    this.totalNodeCapacity = 0;
    this.numContainers = 0;

    for (RMNode ni : rmContext.getRMNodes().values()) {
      SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
      this.usedNodeCapacity += report.getUsedResource().getMemory();
      this.availNodeCapacity += report.getAvailableResource().getMemory();
      this.totalNodeCapacity += ni.getTotalCapability().getMemory();
      this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
    }
  }
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public UserMetricsInfo(final ResourceManager rm, final String user) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  QueueMetrics userMetrics = metrics.getUserMetrics(user);
  this.userMetricsAvailable = false;

  if (userMetrics != null) {
    this.userMetricsAvailable = true;

    this.appsSubmitted = userMetrics.getAppsSubmitted();
    this.appsCompleted = userMetrics.getAppsCompleted();
    this.appsPending = userMetrics.getAppsPending();
    this.appsRunning = userMetrics.getAppsRunning();
    this.appsFailed = userMetrics.getAppsFailed();
    this.appsKilled = userMetrics.getAppsKilled();

    this.runningContainers = userMetrics.getAllocatedContainers();
    this.pendingContainers = userMetrics.getPendingContainers();
    this.reservedContainers = userMetrics.getReservedContainers();

    this.reservedMB = userMetrics.getReservedMB();
    this.pendingMB = userMetrics.getPendingMB();
    this.allocatedMB = userMetrics.getAllocatedMB();

    this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
    this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
    this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();

    this.reservedGpuCores = userMetrics.getReservedGpuCores();
    this.pendingGpuCores = userMetrics.getPendingGpuCores();
    this.allocatedGpuCores = userMetrics.getAllocatedGpuCores();
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.reservedGpuCores = metrics.getReservedGpuCores();
  this.availableGpuCores = metrics.getAvailableGpuCores();
  this.allocatedGpuCores = metrics.getAllocatedGpuCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.totalGpuCores = availableGpuCores + allocatedGpuCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
@Inject QueuesBlock(ResourceManager rm, CSQInfo info) {
  cs = (CapacityScheduler) rm.getResourceScheduler();
  csqinfo = info;
  this.rm = rm;
  RMNodeLabelsManager nodeLabelManager =
      rm.getRMContext().getNodeLabelManager();
  nodeLabelsInfo = nodeLabelManager.pullRMNodeLabelsInfo();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public UserMetricsInfo(final ResourceManager rm, final String user) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  QueueMetrics userMetrics = metrics.getUserMetrics(user);
  this.userMetricsAvailable = false;

  if (userMetrics != null) {
    this.userMetricsAvailable = true;

    this.appsSubmitted = userMetrics.getAppsSubmitted();
    this.appsCompleted = userMetrics.getAppsCompleted();
    this.appsPending = userMetrics.getAppsPending();
    this.appsRunning = userMetrics.getAppsRunning();
    this.appsFailed = userMetrics.getAppsFailed();
    this.appsKilled = userMetrics.getAppsKilled();

    this.runningContainers = userMetrics.getAllocatedContainers();
    this.pendingContainers = userMetrics.getPendingContainers();
    this.reservedContainers = userMetrics.getReservedContainers();

    this.reservedMB = userMetrics.getReservedMB();
    this.pendingMB = userMetrics.getPendingMB();
    this.allocatedMB = userMetrics.getAllocatedMB();

    this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
    this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
    this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissioningNodes = clusterMetrics.getNumDecommissioningNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.shutdownNodes = clusterMetrics.getNumShutdownNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes + shutdownNodes;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
public ClusterMetricsInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();

  this.appsSubmitted = metrics.getAppsSubmitted();
  this.appsCompleted = metrics.getAppsCompleted();
  this.appsPending = metrics.getAppsPending();
  this.appsRunning = metrics.getAppsRunning();
  this.appsFailed = metrics.getAppsFailed();
  this.appsKilled = metrics.getAppsKilled();

  this.reservedMB = metrics.getReservedMB();
  this.availableMB = metrics.getAvailableMB();
  this.allocatedMB = metrics.getAllocatedMB();

  this.reservedVirtualCores = metrics.getReservedVirtualCores();
  this.availableVirtualCores = metrics.getAvailableVirtualCores();
  this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();

  this.containersAllocated = metrics.getAllocatedContainers();
  this.containersPending = metrics.getPendingContainers();
  this.containersReserved = metrics.getReservedContainers();

  this.totalMB = availableMB + allocatedMB;
  this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
  this.activeNodes = clusterMetrics.getNumActiveNMs();
  this.lostNodes = clusterMetrics.getNumLostNMs();
  this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
  this.decommissionedNodes = clusterMetrics.getNumDecommisionedNMs();
  this.rebootedNodes = clusterMetrics.getNumRebootedNMs();
  this.totalNodes = activeNodes + lostNodes + decommissionedNodes
      + rebootedNodes + unhealthyNodes;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
@Inject QueuesBlock(ResourceManager rm) {
  sinfo = new FifoSchedulerInfo(rm);
  fs = (FifoScheduler) rm.getResourceScheduler();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
@Inject QueuesBlock(ResourceManager rm, FSQInfo info) {
  fs = (FairScheduler)rm.getResourceScheduler();
  fsqinfo = info;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; //导入方法依赖的package包/类
@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
  resourceManager = new ResourceManager() {
     @Override
      protected RMNodeLabelsManager createNodeLabelManager() {
        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
        mgr.init(getConfig());
        return mgr;
      }
  };
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  StringBuilder qState = new StringBuilder();
  qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
      .append(CapacitySchedulerConfiguration.DOT)
      .append(CapacitySchedulerConfiguration.STATE);
  csConf.set(qState.toString(), QueueState.STOPPED.name());
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
      .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());

  ResourceScheduler scheduler = resourceManager.getResourceScheduler();

  // Register node1
  String host_0 = "host_0";
  NodeManager nm_0 =
      registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
          Resources.createResource(6 * GB, 1));

  // ResourceRequest priorities
  Priority priority_0 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(0);
  Priority priority_1 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(1);

  // Submit application_0
  Application application_0 =
      new Application("user_0", "a1", resourceManager);
  application_0.submit(); // app + app attempt event sent to scheduler

  application_0.addNodeManager(host_0, 1234, nm_0);

  Resource capability_0_0 = Resources.createResource(3 * GB, 1);
  application_0.addResourceRequestSpec(priority_1, capability_0_0);

  Resource capability_0_1 = Resources.createResource(2 * GB, 1);
  application_0.addResourceRequestSpec(priority_0, capability_0_1);

  Task task_0_0 =
      new Task(application_0, priority_1, new String[] { host_0 });
  application_0.addTask(task_0_0);

  // Send resource requests to the scheduler
  application_0.schedule(); // allocate

  // task_0_0 allocated
  nodeUpdate(nm_0);

  // Get allocations from the scheduler
  application_0.schedule(); // task_0_0
  checkApplicationResourceUsage(3 * GB, application_0);

  checkNodeResourceUsage(3 * GB, nm_0);
  // b2 queue contains 3GB consumption app,
  // add another 3GB will hit max capacity limit on queue b
  scheduler.moveApplication(application_0.getApplicationId(), "b1");

}