Java CapacityScheduler类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  configuration = new YarnConfiguration();
  configuration.set(YarnConfiguration.RM_SCHEDULER,
      CapacityScheduler.class.getCanonicalName());
  fs = FileSystem.get(configuration);
  workingPath =
      new Path(new File("target", this.getClass().getSimpleName()
          + "-remoteDir").getAbsolutePath());
  configuration.set(YarnConfiguration.FS_BASED_RM_CONF_STORE,
      workingPath.toString());
  tmpDir = new Path(new File("target", this.getClass().getSimpleName()
      + "-tmpDir").getAbsolutePath());
  fs.delete(workingPath, true);
  fs.delete(tmpDir, true);
  fs.mkdirs(workingPath);
  fs.mkdirs(tmpDir);

  // reset the groups to what it default test settings
  MockUnixGroupsMapping.resetGroups();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
@Test
public void testAdminRefreshQueuesWithLocalConfigurationProvider()
    throws IOException, YarnException {
  rm = new MockRM(configuration);
  rm.init(configuration);
  rm.start();

  CapacityScheduler cs =
      (CapacityScheduler) rm.getRMContext().getScheduler();
  int maxAppsBefore = cs.getConfiguration().getMaximumSystemApplications();

  try {
    rm.adminService.refreshQueues(RefreshQueuesRequest.newInstance());
    Assert.assertEquals(maxAppsBefore, cs.getConfiguration()
        .getMaximumSystemApplications());
  } catch (Exception ex) {
    fail("Using localConfigurationProvider. Should not get any exception.");
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public CapacityScheduler mockCapacityScheduler(int numContainers)
    throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
  cs.setConf(new YarnConfiguration());

  RMContext mockRmContext = createRMContext(conf);

  cs.setRMContext(mockRmContext);
  try {
    cs.serviceInit(conf);
  } catch (Exception e) {
    Assert.fail(e.getMessage());
  }

  initializeRMContext(numContainers, cs, mockRmContext);
  return cs;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
CapacitySchedulerHealthInfo(CapacityScheduler cs) {
  SchedulerHealth ht = cs.getSchedulerHealth();
  lastrun = ht.getLastSchedulerRunTime();
  operationsInfo = new HashMap<>();
  operationsInfo.put("last-allocation",
    new OperationInformation(ht.getLastAllocationDetails()));
  operationsInfo.put("last-release",
    new OperationInformation(ht.getLastReleaseDetails()));
  operationsInfo.put("last-preemption",
    new OperationInformation(ht.getLastPreemptionDetails()));
  operationsInfo.put("last-reservation",
    new OperationInformation(ht.getLastReservationDetails()));

  lastRunDetails = new ArrayList<>();
  lastRunDetails.add(new LastRunDetails("releases", ht.getReleaseCount(), ht
    .getResourcesReleased()));
  lastRunDetails.add(new LastRunDetails("allocations", ht
    .getAllocationCount(), ht.getResourcesAllocated()));
  lastRunDetails.add(new LastRunDetails("reservations", ht
    .getReservationCount(), ht.getResourcesReserved()));

}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
  this.maximumClusterPriority =
      rs.getMaxClusterLevelAppPriority().getPriority();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
private CapacityScheduler initializeCapacityScheduler() {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  ReservationSystemTestUtil.setupQueueConfiguration(conf);

  CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
  cs.setConf(conf);

  mockRMContext = ReservationSystemTestUtil.createRMContext(conf);

  cs.setRMContext(mockRMContext);
  try {
    cs.serviceInit(conf);
  } catch (Exception e) {
    Assert.fail(e.getMessage());
  }
  ReservationSystemTestUtil.initializeRMContext(10, cs, mockRMContext);
  return cs;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
public CapacityScheduler mockCapacityScheduler(int numContainers)
    throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = Mockito.spy(new CapacityScheduler());
  cs.setConf(new YarnConfiguration());

  RMContext mockRmContext = createRMContext(conf);

  cs.setRMContext(mockRmContext);
  try {
    cs.serviceInit(conf);
  } catch (Exception e) {
    Assert.fail(e.getMessage());
  }

  initializeRMContext(numContainers, cs, mockRmContext);
  return cs;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
public void setup() {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };

  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
    .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
private void startRM() throws IOException, ClassNotFoundException {
  Configuration rmConf = new YarnConfiguration();
  String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);

  // For CapacityScheduler we use a sub-classing instead of wrapping
  // to allow scheduler-specific invocations from monitors to work
  // this can be used for other schedulers as well if we care to
  // exercise/track behaviors that are not common to the scheduler api
  if(Class.forName(schedulerClass) == CapacityScheduler.class) {
    rmConf.set(YarnConfiguration.RM_SCHEDULER,
        SLSCapacityScheduler.class.getName());
  } else {
    rmConf.set(YarnConfiguration.RM_SCHEDULER,
            ResourceSchedulerWrapper.class.getName());
    rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
  }

  rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
  rm = new ResourceManager();
  rm.init(rmConf);
  rm.start();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
public static CapacityScheduler mockCapacityScheduler() throws IOException {
  // stolen from TestCapacityScheduler
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);

  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  RMContext rmContext = new RMContextImpl(null, null, null, null, null,
      null, new RMContainerTokenSecretManager(conf),
      new NMTokenSecretManagerInRM(conf),
      new ClientToAMTokenSecretManagerInRM(), null);
  rmContext.setNodeLabelManager(new MemoryRMNodeLabelsManager());
  cs.setRMContext(rmContext);
  cs.init(conf);
  return cs;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  csConf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
  ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  bind(RMContext.class).toInstance(rm.getRMContext());
  bind(ApplicationACLsManager.class).toInstance(
      rm.getApplicationACLsManager());
  bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager());
  serve("/*").with(GuiceContainer.class);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; //导入依赖的package包/类
@Before
public void setup() throws Exception {

  // 1 sec step
  step = 1000L;

  initTime = System.currentTimeMillis();
  minAlloc = Resource.newInstance(1024, 1);
  res = new DefaultResourceCalculator();
  maxAlloc = Resource.newInstance(1024 * 8, 8);

  mAgent = mock(ReservationAgent.class);
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  CapacityScheduler scheduler = testUtil.mockCapacityScheduler(totCont);
  String reservationQ = testUtil.getFullReservationQueueName();
  CapacitySchedulerConfiguration capConf = scheduler.getConfiguration();
  NoOverCommitPolicy policy = new NoOverCommitPolicy();
  policy.init(reservationQ, capConf);

  plan =
      new InMemoryPlan(scheduler.getRootQueueMetrics(), policy, mAgent,
          scheduler.getClusterResource(), step, res, minAlloc, maxAlloc,
          "dedicated", null, true);
}