Java MockRM.getResourceScheduler方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Before
public void setup() {
  mockClock = new MockClock();
  conf = createConfiguration();
  resourceManager = new MockRM(conf);
  resourceManager.start();

  scheduler = (FairScheduler) resourceManager.getResourceScheduler();
  scheduler.setClock(mockClock);

  assertTrue(scheduler.isContinuousSchedulingEnabled());
  assertEquals(
      FairSchedulerConfiguration.DEFAULT_CONTINUOUS_SCHEDULING_SLEEP_MS,
      scheduler.getContinuousSchedulingSleepMs());
  assertEquals(mockClock, scheduler.getClock());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void checkAppQueue(MockRM resourceManager, String user,
    String submissionQueue, String expected)
    throws Exception {
  RMApp app = resourceManager.submitApp(200, "name", user,
      new HashMap<ApplicationAccessType, String>(), false, submissionQueue, -1,
      null, "MAPREDUCE", false);
  RMAppState expectedState = expected.isEmpty() ? RMAppState.FAILED
      : RMAppState.ACCEPTED;
  resourceManager.waitForState(app.getApplicationId(), expectedState);
  // get scheduler app
  CapacityScheduler cs = (CapacityScheduler)
      resourceManager.getResourceScheduler();
  SchedulerApplication schedulerApp =
      cs.getSchedulerApplications().get(app.getApplicationId());
  String queue = "";
  if (schedulerApp != null) {
    queue = schedulerApp.getQueue().getQueueName();
  }
  Assert.assertTrue("expected " + expected + " actual " + queue,
      expected.equals(queue));
  Assert.assertEquals(expected, app.getQueue());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testUpdateDemand() {
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();
  scheduler.allocConf = mock(AllocationConfiguration.class);

  String queueName = "root.queue1";
  when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
  when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
  FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);

  FSAppAttempt app = mock(FSAppAttempt.class);
  Mockito.when(app.getDemand()).thenReturn(maxResource);

  schedulable.addAppSchedulable(app);
  schedulable.addAppSchedulable(app);

  schedulable.updateDemand();

  assertTrue("Demand is greater than max allowed ",
      Resources.equals(schedulable.getDemand(), maxResource));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testExistenceOfResourceRequestInRMContainer() throws Exception {
  Configuration conf = new Configuration();
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
  RMApp app1 = rm1.submitApp(1024);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  ResourceScheduler scheduler = rm1.getResourceScheduler();

  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 = ContainerId.newContainerId(
      am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

  // Verify whether list of ResourceRequest is present in RMContainer
  // while moving to ALLOCATED state
  Assert.assertNotNull(scheduler.getRMContainer(containerId2)
      .getResourceRequests());

  // Allocate container
  am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
      .getAllocatedContainers();
  rm1.waitForState(nm1, containerId2, RMContainerState.ACQUIRED);

  // After RMContainer moving to ACQUIRED state, list of ResourceRequest will
  // be empty
  Assert.assertNull(scheduler.getRMContainer(containerId2)
      .getResourceRequests());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void startResourceManager(float utilizationThreshold) {
  conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD,
      utilizationThreshold);
  resourceManager = new MockRM(conf);
  resourceManager.start();

  assertTrue(
      resourceManager.getResourceScheduler() instanceof StubbedFairScheduler);
  scheduler = (FairScheduler)resourceManager.getResourceScheduler();

  scheduler.setClock(clock);
  scheduler.updateInterval = 60 * 1000;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void createClusterWithQueuesAndOneNode(int mem, int vCores, int gCores,
    String policy) throws IOException {
  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"root\" >");
  out.println("   <queue name=\"parentA\" >");
  out.println("       <weight>8</weight>");
  out.println("       <queue name=\"childA1\" />");
  out.println("       <queue name=\"childA2\" />");
  out.println("       <queue name=\"childA3\" />");
  out.println("       <queue name=\"childA4\" />");
  out.println("   </queue>");
  out.println("   <queue name=\"parentB\" >");
  out.println("       <weight>1</weight>");
  out.println("       <queue name=\"childB1\" />");
  out.println("       <queue name=\"childB2\" />");
  out.println("   </queue>");
  out.println("</queue>");
  out.println("<defaultQueueSchedulingPolicy>" + policy
      + "</defaultQueueSchedulingPolicy>");
  out.println("</allocations>");
  out.close();

  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();

  RMNode node1 = MockNodes.newNodeInfo(1,
      Resources.createResource(mem, vCores, gCores), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void checkPendingResource(MockRM rm, String queueName, int memory,
    String label) {
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
  CSQueue queue = cs.getQueue(queueName);
  Assert.assertEquals(
      memory,
      queue.getQueueResourceUsage()
          .getPending(label == null ? RMNodeLabelsManager.NO_LABEL : label)
          .getMemory());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void checkPendingResourceGreaterThanZero(MockRM rm, String queueName,
    String label) {
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
  CSQueue queue = cs.getQueue(queueName);
  Assert.assertTrue(queue.getQueueResourceUsage()
      .getPending(label == null ? RMNodeLabelsManager.NO_LABEL : label)
      .getMemory() > 0);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void checkUsedResource(MockRM rm, String queueName, int memory,
    String label) {
  CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
  CSQueue queue = scheduler.getQueue(queueName);
  Assert.assertEquals(memory, queue.getQueueResourceUsage().getUsed(label)
      .getMemory());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void sentRMContainerLaunched(MockRM rm, ContainerId containerId) {
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
  RMContainer rmContainer = cs.getRMContainer(containerId);
  if (rmContainer != null) {
    rmContainer.handle(
        new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
  } else {
    Assert.fail("Cannot find RMContainer");
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testUpdatePriorityAtRuntime() throws Exception {

  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  // Set Max Application Priority as 10
  conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
  MockRM rm = new MockRM(conf);
  rm.start();

  Priority appPriority1 = Priority.newInstance(5);
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB);
  RMApp app1 = rm.submitApp(1 * GB, appPriority1);

  // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
  MockAM am1 = MockRM.launchAM(app1, rm, nm1);
  am1.registerAppAttempt();

  // get scheduler
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  // Change the priority of App1 to 8
  Priority appPriority2 = Priority.newInstance(8);
  cs.updateApplicationPriority(appPriority2, app1.getApplicationId());

  // get scheduler app
  FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications()
      .get(app1.getApplicationId()).getCurrentAppAttempt();

  // Verify whether the new priority is updated
  Assert.assertEquals(appPriority2, schedulerAppAttempt.getPriority());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void doNMHeartbeat(MockRM rm, NodeId nodeId, int nHeartbeat) {
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
  RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nodeId);
  for (int i = 0; i < nHeartbeat; i++) {
    cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 20000)
public void testPreemptedAMRestartOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  CapacityScheduler scheduler =
      (CapacityScheduler) rm1.getResourceScheduler();
  ContainerId amContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

  // Forcibly preempt the am container;
  scheduler.killContainer(scheduler.getRMContainer(amContainer));

  am1.waitForState(RMAppAttemptState.FAILED);
  Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry());
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // state store has 1 attempt stored.
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  Assert.assertEquals(1, appState.getAttemptCount());
  // attempt stored has the preempted container exit status.
  Assert.assertEquals(ContainerExitStatus.PREEMPTED,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  nm1.registerNode();
  rm2.start();

  // Restarted RM should re-launch the am.
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt2 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt2.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());
  rm1.stop();
  rm2.stop();
}