Java FairScheduler类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Inject
public FairSchedulerAppsBlock(ResourceManager rm, ViewContext ctx,
    Configuration conf) {
  super(ctx);
  FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
  fsinfo = new FairSchedulerInfo(scheduler);
  apps = new ConcurrentHashMap<ApplicationId, RMApp>();
  for (Map.Entry<ApplicationId, RMApp> entry : rm.getRMContext().getRMApps()
      .entrySet()) {
    if (!(RMAppState.NEW.equals(entry.getValue().getState())
        || RMAppState.NEW_SAVING.equals(entry.getValue().getState())
        || RMAppState.SUBMITTED.equals(entry.getValue().getState()))) {
      apps.put(entry.getKey(), entry.getValue());
    }
  }
  this.conf = conf;
  this.rm = rm;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
public void scheduler() {
  // limit applications to those in states relevant to scheduling
  set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.NEW.toString(),
      YarnApplicationState.NEW_SAVING.toString(),
      YarnApplicationState.SUBMITTED.toString(),
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));

  ResourceManager rm = getInstance(ResourceManager.class);
  ResourceScheduler rs = rm.getResourceScheduler();
  if (rs == null || rs instanceof CapacityScheduler) {
    setTitle("Capacity Scheduler");
    render(CapacitySchedulerPage.class);
    return;
  }
  
  if (rs instanceof FairScheduler) {
    setTitle("Fair Scheduler");
    render(FairSchedulerPage.class);
    return;
  }
  
  setTitle("Default Scheduler");
  render(DefaultSchedulerPage.class);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
private void configureFairScheduler(YarnConfiguration conf) throws IOException {
  // Disable queueMaxAMShare limitation for fair scheduler
  PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
  out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
  out.println("<queue name=\"root\">");
  out.println("  <schedulingPolicy>drf</schedulingPolicy>");
  out.println("  <weight>1.0</weight>");
  out.println("  <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
  out.println("  <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
  out.println("  <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"test\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch(IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
public static FairScheduler setupFairScheduler(
    ReservationSystemTestUtil testUtil,
    RMContext rmContext, Configuration conf, int numContainers) throws
    IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource = testUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
protected FairSchedulerQueueInfoList getChildQueues(FSQueue queue,
                                                    FairScheduler scheduler) {
  // Return null to omit 'childQueues' field from the return value of
  // REST API if it is empty. We omit the field to keep the consistency
  // with CapacitySchedulerQueueInfo, which omits 'queues' field if empty.
  Collection<FSQueue> children = queue.getChildQueues();
  if (children.isEmpty()) {
    return null;
  }
  FairSchedulerQueueInfoList list = new FairSchedulerQueueInfoList();
  for (FSQueue child : children) {
    if (child instanceof FSLeafQueue) {
      list.addToQueueInfoList(
          new FairSchedulerLeafQueueInfo((FSLeafQueue) child, scheduler));
    } else {
      list.addToQueueInfoList(
          new FairSchedulerQueueInfo(child, scheduler));
    }
  }
  return list;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
public SchedulerInfo(final ResourceManager rm) {
  ResourceScheduler rs = rm.getResourceScheduler();

  if (rs instanceof CapacityScheduler) {
    this.schedulerName = "Capacity Scheduler";
  } else if (rs instanceof FairScheduler) {
    this.schedulerName = "Fair Scheduler";
  } else if (rs instanceof FifoScheduler) {
    this.schedulerName = "Fifo Scheduler";
  }
  this.minAllocResource = new ResourceInfo(rs.getMinimumResourceCapability());
  this.maxAllocResource = new ResourceInfo(rs.getMaximumResourceCapability());
  this.schedulingResourceTypes = rs.getSchedulingResourceTypes();
  this.maximumClusterPriority =
      rs.getMaxClusterLevelAppPriority().getPriority();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Test
public void testEmptyChildQueues() throws Exception {
  FairSchedulerConfiguration conf = new FairSchedulerConfiguration();
  FairScheduler scheduler = mock(FairScheduler.class);
  AllocationConfiguration allocConf = new AllocationConfiguration(conf);
  when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
  when(scheduler.getConf()).thenReturn(conf);
  when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(1, 1));
  SystemClock clock = new SystemClock();
  when(scheduler.getClock()).thenReturn(clock);
  QueueManager queueManager = new QueueManager(scheduler);
  queueManager.initialize(conf);

  FSQueue testQueue = queueManager.getLeafQueue("test", true);
  FairSchedulerQueueInfo queueInfo =
      new FairSchedulerQueueInfo(testQueue, scheduler);
  Collection<FairSchedulerQueueInfo> childQueues =
      queueInfo.getChildQueues();
  Assert.assertNotNull(childQueues);
  Assert.assertEquals("Child QueueInfo was not empty", 0, childQueues.size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("  <queue name=\"dedicated\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch (IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
public static FairScheduler setupFairScheduler(RMContext rmContext,
    Configuration conf, int numContainers) throws IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource =
      ReservationSystemTestUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
private Configuration getSchedulerDynamicConfiguration() throws IOException {
  conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
  conf.setTimeDuration(
      YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER_TIME_STEP, 1L,
      TimeUnit.SECONDS);
  if (getSchedulerType() == SchedulerType.CAPACITY) {
    CapacitySchedulerConfiguration schedulerConf =
        new CapacitySchedulerConfiguration(conf);
    ReservationSystemTestUtil.setupDynamicQueueConfiguration(schedulerConf);
    return schedulerConf;
  } else {
    String allocFile = new File(FairSchedulerTestBase.TEST_DIR,
        TestWorkPreservingRMRestart.class.getSimpleName() + ".xml")
            .getAbsolutePath();
    ReservationSystemTestUtil.setupFSAllocationFile(allocFile);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
        ResourceScheduler.class);
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
    return conf;
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  bind(RMContext.class).toInstance(rm.getRMContext());
  bind(ApplicationACLsManager.class).toInstance(
      rm.getApplicationACLsManager());
  bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager());
  serve("/*").with(GuiceContainer.class);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Override
public void configureScheduler() {
  try {
    PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"root\">");
    out.println("  <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  <queue name=\"default\">");
    out.println("    <aclAdministerApps>someuser </aclAdministerApps>");
    out.println("  </queue>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
  } catch(IOException e) {
  }
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
  conf.set("yarn.resourcemanager.scheduler.class",
      FairScheduler.class.getName());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Override
protected void configureServlets() {
  try {
    bind(JAXBContextResolver.class);
    bind(RMWebServices.class);
    bind(GenericExceptionHandler.class);
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
        ResourceScheduler.class);
    RMStorageFactory.setConfiguration(conf);
    YarnAPIStorageFactory.setConfiguration(conf);
    DBUtility.InitializeDB();
    rm = new MockRM(conf);
    bind(ResourceManager.class).toInstance(rm);
    serve("/*").with(GuiceContainer.class);
  } catch (IOException ex) {
    Logger.getLogger(TestRMWebServicesFairScheduler.class.getName()).log(Level.SEVERE, null, ex);
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; //导入依赖的package包/类
@Test (timeout = 30000)
public void testAppWithNoContainers() throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  MockRM rm = new MockRM(conf);
  Assume.assumeFalse(rm.getResourceScheduler() instanceof FairScheduler);
  rm.start();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  
  RMApp app = rm.submitApp(2000);

  //kick the scheduling
  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  am.unregisterAppAttempt();
  nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.stop();
}