Java RMAppAttemptState.FINISHED属性代码示例

@Test
public void testAppAdditionAndRemoval() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  ApplicationAttemptId attemptId =createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default",
    "user1");
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attemptAddedEvent =
      new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
  scheduler.handle(attemptAddedEvent);

  // Scheduler should have two queues (the default and the one created for user1)
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());

  // That queue should have one app
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
      createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);

  // Now remove app
  scheduler.handle(appRemovedEvent1);

  // Queue should have no apps
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());
}
 

@Test
public void testMaxRunningAppsHierarchicalQueues() throws Exception {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  MockClock clock = new MockClock();
  scheduler.setClock(clock);

  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"queue1\">");
  out.println("  <maxRunningApps>3</maxRunningApps>");
  out.println("  <queue name=\"sub1\"></queue>");
  out.println("  <queue name=\"sub2\"></queue>");
  out.println("  <queue name=\"sub3\">");
  out.println("    <maxRunningApps>1</maxRunningApps>");
  out.println("  </queue>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // exceeds no limits
  ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1");
  verifyAppRunnable(attId1, true);
  verifyQueueNumRunnable("queue1.sub1", 1, 0);
  clock.tick(10);
  // exceeds no limits
  ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1.sub3", "user1");
  verifyAppRunnable(attId2, true);
  verifyQueueNumRunnable("queue1.sub3", 1, 0);
  clock.tick(10);
  // exceeds no limits
  ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1.sub2", "user1");
  verifyAppRunnable(attId3, true);
  verifyQueueNumRunnable("queue1.sub2", 1, 0);
  clock.tick(10);
  // exceeds queue1 limit
  ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1.sub2", "user1");
  verifyAppRunnable(attId4, false);
  verifyQueueNumRunnable("queue1.sub2", 1, 1);
  clock.tick(10);
  // exceeds sub3 limit
  ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1.sub3", "user1");
  verifyAppRunnable(attId5, false);
  verifyQueueNumRunnable("queue1.sub3", 1, 1);
  clock.tick(10);
  
  // Even though the app was removed from sub3, the app from sub2 gets to go
  // because it came in first
  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED, false);
  scheduler.handle(appRemovedEvent1);
  verifyAppRunnable(attId4, true);
  verifyQueueNumRunnable("queue1.sub2", 2, 0);
  verifyAppRunnable(attId5, false);
  verifyQueueNumRunnable("queue1.sub3", 0, 1);

  // Now test removal of a non-runnable app
  AppAttemptRemovedSchedulerEvent appRemovedEvent2 =
      new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED, true);
  scheduler.handle(appRemovedEvent2);
  assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps
      .get("user1").size());
  // verify app gone in queue accounting
  verifyQueueNumRunnable("queue1.sub3", 0, 0);
  // verify it doesn't become runnable when there would be space for it
  AppAttemptRemovedSchedulerEvent appRemovedEvent3 =
      new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED, true);
  scheduler.handle(appRemovedEvent3);
  verifyQueueNumRunnable("queue1.sub2", 1, 0);
  verifyQueueNumRunnable("queue1.sub3", 0, 0);
}
 

@Test
public void testAppAdditionAndRemoval() throws Exception {
  ApplicationAttemptId attemptId = createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default",
          "user1");
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attemptAddedEvent =
      new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
  scheduler.handle(attemptAddedEvent);

  // Scheduler should have two queues (the default and the one created for
  // user1)
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());

  // That queue should have one app
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(createAppAttemptId(1, 1),
          RMAppAttemptState.FINISHED, false);

  // Now remove app
  scheduler.handle(appRemovedEvent1);

  // Queue should have no apps
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());
}
 

@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {

  RMAppRecoverEvent recoverEvent = (RMAppRecoverEvent) event;
  app.recover(recoverEvent.getRMState());
  // The app has completed.
  if (app.recoveredFinalState != null) {
    app.recoverAppAttempts();
    new FinalTransition(app.recoveredFinalState).transition(app, event);
    return app.recoveredFinalState;
  }

  if (UserGroupInformation.isSecurityEnabled()) {
    // synchronously renew delegation token on recovery.
    try {
      app.rmContext.getDelegationTokenRenewer().addApplicationSync(
        app.getApplicationId(), app.parseCredentials(),
        app.submissionContext.getCancelTokensWhenComplete(), app.getUser());
    } catch (Exception e) {
      String msg = "Failed to renew token for " + app.applicationId
              + " on recovery : " + e.getMessage();
      app.diagnostics.append(msg);
      LOG.error(msg, e);
    }
  }

  // No existent attempts means the attempt associated with this app was not
  // started or started but not yet saved.
  if (app.attempts.isEmpty()) {
    app.scheduler.handle(new AppAddedSchedulerEvent(app.applicationId,
      app.submissionContext.getQueue(), app.user,
      app.submissionContext.getReservationID()));
    return RMAppState.SUBMITTED;
  }

  // Add application to scheduler synchronously to guarantee scheduler
  // knows applications before AM or NM re-registers.
  app.scheduler.handle(new AppAddedSchedulerEvent(app.applicationId,
    app.submissionContext.getQueue(), app.user, true,
      app.submissionContext.getReservationID()));

  // recover attempts
  app.recoverAppAttempts();

  // Last attempt is in final state, return ACCEPTED waiting for last
  // RMAppAttempt to send finished or failed event back.
  if (app.currentAttempt != null
      && (app.currentAttempt.getState() == RMAppAttemptState.KILLED
          || app.currentAttempt.getState() == RMAppAttemptState.FINISHED
          || (app.currentAttempt.getState() == RMAppAttemptState.FAILED
              && app.getNumFailedAppAttempts() == app.maxAppAttempts))) {
    return RMAppState.ACCEPTED;
  }

  // YARN-1507 is saving the application state after the application is
  // accepted. So after YARN-1507, an app is saved meaning it is accepted.
  // Thus we return ACCECPTED state on recovery.
  return RMAppState.ACCEPTED;
}
 

@Test
public void testUserAndQueueMaxRunningApps() throws Exception {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);

  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"queue1\">");
  out.println("<maxRunningApps>2</maxRunningApps>");
  out.println("</queue>");
  out.println("<user name=\"user1\">");
  out.println("<maxRunningApps>1</maxRunningApps>");
  out.println("</user>");
  out.println("</allocations>");
  out.close();

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // exceeds no limits
  ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1", "user1");
  verifyAppRunnable(attId1, true);
  verifyQueueNumRunnable("queue1", 1, 0);
  // exceeds user limit
  ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue2", "user1");
  verifyAppRunnable(attId2, false);
  verifyQueueNumRunnable("queue2", 0, 1);
  // exceeds no limits
  ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1", "user2");
  verifyAppRunnable(attId3, true);
  verifyQueueNumRunnable("queue1", 2, 0);
  // exceeds queue limit
  ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1", "user2");
  verifyAppRunnable(attId4, false);
  verifyQueueNumRunnable("queue1", 2, 1);
  
  // Remove app 1 and both app 2 and app 4 should becomes runnable in its place
  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED, false);
  scheduler.handle(appRemovedEvent1);
  verifyAppRunnable(attId2, true);
  verifyQueueNumRunnable("queue2", 1, 0);
  verifyAppRunnable(attId4, true);
  verifyQueueNumRunnable("queue1", 2, 0);
  
  // A new app to queue1 should not be runnable
  ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1", "user2");
  verifyAppRunnable(attId5, false);
  verifyQueueNumRunnable("queue1", 2, 1);
}
 

@Test
public void testSchedulingOnRemovedNode() throws Exception {
  // Disable continuous scheduling, will invoke continuous scheduling manually
  scheduler.init(conf);
  scheduler.start();
  Assert.assertTrue("Continuous scheduling should be disabled.",
      !scheduler.isContinuousSchedulingEnabled());

  ApplicationAttemptId id11 = createAppAttemptId(1, 1);
  createMockRMApp(id11);

  scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1",
      false);
  scheduler.addApplicationAttempt(id11, false, false);

  List<ResourceRequest> ask1 = new ArrayList<>();
  ResourceRequest request1 =
      createResourceRequest(1024, 8, 8, ResourceRequest.ANY, 1, 1, true);

  ask1.add(request1);
  scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null,
      null);

  String hostName = "127.0.0.1";
  RMNode node1 = MockNodes.newNodeInfo(1,
    Resources.createResource(8 * 1024, 8, 8), 1, hostName);
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  FSSchedulerNode node = (FSSchedulerNode)scheduler.getSchedulerNode(
    node1.getNodeID());

  NodeRemovedSchedulerEvent removeNode1 =
      new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(removeNode1);

  scheduler.attemptScheduling(node);

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(id11,
          RMAppAttemptState.FINISHED, false);
  scheduler.handle(appRemovedEvent1);
}
 

@Test
public void testFairShareResetsToZeroWhenAppsComplete() throws IOException {
  int nodeCapacity = 16 * 1024;
  createClusterWithQueuesAndOneNode(nodeCapacity, "fair");

  // Run apps in childA1,childA2 which are under parentA
  ApplicationAttemptId app1 = createSchedulingRequest(2 * 1024,
      "root.parentA.childA1", "user1");
  ApplicationAttemptId app2 = createSchedulingRequest(3 * 1024,
      "root.parentA.childA2", "user2");

  scheduler.update();

  // Verify if both the active queues under parentA get 50% fair
  // share
  for (int i = 1; i <= 2; i++) {
    assertEquals(
        50,
        (double) scheduler.getQueueManager()
            .getLeafQueue("root.parentA.childA" + i, false).getFairShare()
            .getMemory()
            / nodeCapacity * 100, .9);
  }
  // Let app under childA1 complete. This should cause the fair share
  // of queue childA1 to be reset to zero,since the queue has no apps running.
  // Queue childA2's fair share would increase to 100% since its the only
  // active queue.
  AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
      app1, RMAppAttemptState.FINISHED, false);

  scheduler.handle(appRemovedEvent1);
  scheduler.update();

  assertEquals(
      0,
      (double) scheduler.getQueueManager()
          .getLeafQueue("root.parentA.childA1", false).getFairShare()
          .getMemory()
          / nodeCapacity * 100, 0);
  assertEquals(
      100,
      (double) scheduler.getQueueManager()
          .getLeafQueue("root.parentA.childA2", false).getFairShare()
          .getMemory()
          / nodeCapacity * 100, 0.1);

  verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
      nodeCapacity);
}
 

@Test
public void testMaxRunningAppsHierarchicalQueues() throws Exception {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  ControlledClock clock = new ControlledClock();
  scheduler.setClock(clock);

  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"queue1\">");
  out.println("  <maxRunningApps>3</maxRunningApps>");
  out.println("  <queue name=\"sub1\"></queue>");
  out.println("  <queue name=\"sub2\"></queue>");
  out.println("  <queue name=\"sub3\">");
  out.println("    <maxRunningApps>1</maxRunningApps>");
  out.println("  </queue>");
  out.println("</queue>");
  out.println("</allocations>");
  out.close();

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // exceeds no limits
  ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue1.sub1", "user1");
  verifyAppRunnable(attId1, true);
  verifyQueueNumRunnable("queue1.sub1", 1, 0);
  clock.tickSec(10);
  // exceeds no limits
  ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1.sub3", "user1");
  verifyAppRunnable(attId2, true);
  verifyQueueNumRunnable("queue1.sub3", 1, 0);
  clock.tickSec(10);
  // exceeds no limits
  ApplicationAttemptId attId3 = createSchedulingRequest(1024, "queue1.sub2", "user1");
  verifyAppRunnable(attId3, true);
  verifyQueueNumRunnable("queue1.sub2", 1, 0);
  clock.tickSec(10);
  // exceeds queue1 limit
  ApplicationAttemptId attId4 = createSchedulingRequest(1024, "queue1.sub2", "user1");
  verifyAppRunnable(attId4, false);
  verifyQueueNumRunnable("queue1.sub2", 1, 1);
  clock.tickSec(10);
  // exceeds sub3 limit
  ApplicationAttemptId attId5 = createSchedulingRequest(1024, "queue1.sub3", "user1");
  verifyAppRunnable(attId5, false);
  verifyQueueNumRunnable("queue1.sub3", 1, 1);
  clock.tickSec(10);

  // Even though the app was removed from sub3, the app from sub2 gets to go
  // because it came in first
  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED, false);
  scheduler.handle(appRemovedEvent1);
  verifyAppRunnable(attId4, true);
  verifyQueueNumRunnable("queue1.sub2", 2, 0);
  verifyAppRunnable(attId5, false);
  verifyQueueNumRunnable("queue1.sub3", 0, 1);

  // Now test removal of a non-runnable app
  AppAttemptRemovedSchedulerEvent appRemovedEvent2 =
      new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED, true);
  scheduler.handle(appRemovedEvent2);
  assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps
      .get("user1").size());
  // verify app gone in queue accounting
  verifyQueueNumRunnable("queue1.sub3", 0, 0);
  // verify it doesn't become runnable when there would be space for it
  AppAttemptRemovedSchedulerEvent appRemovedEvent3 =
      new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED, true);
  scheduler.handle(appRemovedEvent3);
  verifyQueueNumRunnable("queue1.sub2", 1, 0);
  verifyQueueNumRunnable("queue1.sub3", 0, 0);
}
 

@Test
public void testSchedulingOnRemovedNode() throws Exception {
  // Disable continuous scheduling, will invoke continuous scheduling manually
  scheduler.init(conf);
  scheduler.start();
  Assert.assertTrue("Continuous scheduling should be disabled.",
      !scheduler.isContinuousSchedulingEnabled());

  ApplicationAttemptId id11 = createAppAttemptId(1, 1);
  createMockRMApp(id11);

  scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1",
      false);
  scheduler.addApplicationAttempt(id11, false, false);

  List<ResourceRequest> ask1 = new ArrayList<>();
  ResourceRequest request1 =
      createResourceRequest(1024, 8, ResourceRequest.ANY, 1, 1, true);

  ask1.add(request1);
  scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null,
      null, null, null);

  String hostName = "127.0.0.1";
  RMNode node1 = MockNodes.newNodeInfo(1,
    Resources.createResource(8 * 1024, 8), 1, hostName);
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  FSSchedulerNode node = (FSSchedulerNode)scheduler.getSchedulerNode(
    node1.getNodeID());

  NodeRemovedSchedulerEvent removeNode1 =
      new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(removeNode1);

  scheduler.attemptScheduling(node);

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(id11,
          RMAppAttemptState.FINISHED, false);
  scheduler.handle(appRemovedEvent1);
}
 

@Test
public void testSchedulingOnRemovedNode() throws Exception {
  // Disable continuous scheduling, will invoke continuous scheduling manually
  scheduler.init(conf);
  scheduler.start();
  Assert.assertTrue("Continuous scheduling should be disabled.",
      !scheduler.isContinuousSchedulingEnabled());

  ApplicationAttemptId id11 = createAppAttemptId(1, 1);
  createMockRMApp(id11);

  scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1",
      false);
  scheduler.addApplicationAttempt(id11, false, false);

  List<ResourceRequest> ask1 = new ArrayList<>();
  ResourceRequest request1 =
      createResourceRequest(1024, 8, ResourceRequest.ANY, 1, 1, true);

  ask1.add(request1);
  scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null,
      null);

  String hostName = "127.0.0.1";
  RMNode node1 = MockNodes.newNodeInfo(1,
    Resources.createResource(8 * 1024, 8), 1, hostName);
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  FSSchedulerNode node = (FSSchedulerNode)scheduler.getSchedulerNode(
    node1.getNodeID());

  NodeRemovedSchedulerEvent removeNode1 =
      new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(removeNode1);

  scheduler.attemptScheduling(node);

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(id11,
          RMAppAttemptState.FINISHED, false);
  scheduler.handle(appRemovedEvent1);
}