Java AllocateResponse.getCompletedContainersStatuses方法代码示例

import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; //导入方法依赖的package包/类
@Test(timeout = 60000)
public void testProcessingNMContainerStatusesOnNMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // 1. Start the cluster-RM,NM,Submit app with 1024MB,Launch & register AM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  int nmMemory = 8192;
  int amMemory = 1024;
  int containerMemory = 2048;
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", nmMemory, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app0 = rm1.submitApp(amMemory);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  // 2. AM sends ResourceRequest for 1 container with memory 2048MB.
  int noOfContainers = 1;
  List<Container> allocateContainers =
      am0.allocateAndWaitForContainers(noOfContainers, containerMemory, nm1);

  // 3. Verify for number of container allocated by RM
  Assert.assertEquals(noOfContainers, allocateContainers.size());
  Container container = allocateContainers.get(0);

  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), container.getId()
      .getContainerId(), ContainerState.RUNNING);

  rm1.waitForState(app0.getApplicationId(), RMAppState.RUNNING);

  // 4. Verify Memory Usage by cluster, it should be 3072. AM memory +
  // requested memory. 1024 + 2048=3072
  ResourceScheduler rs = rm1.getRMContext().getScheduler();
  int allocatedMB = rs.getRootQueueMetrics().getAllocatedMB();
  Assert.assertEquals(amMemory + containerMemory, allocatedMB);

  // 5. Re-register NM by sending completed container status
  List<NMContainerStatus> nMContainerStatusForApp =
      createNMContainerStatusForApp(am0);
  nm1.registerNode(nMContainerStatusForApp,
      Arrays.asList(app0.getApplicationId()));

  waitForClusterMemory(nm1, rs, amMemory);

  // 6. Verify for Memory Used, it should be 1024
  Assert.assertEquals(amMemory, rs.getRootQueueMetrics().getAllocatedMB());

  // 7. Send AM heatbeat to RM. Allocated response should contain completed
  // container
  AllocateRequest req =
      AllocateRequest.newInstance(0, 0F, new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>(), null);
  AllocateResponse allocate = am0.allocate(req);
  List<ContainerStatus> completedContainersStatuses =
      allocate.getCompletedContainersStatuses();
  Assert.assertEquals(noOfContainers, completedContainersStatuses.size());

  // Application clean up should happen Cluster memory used is 0
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  waitForClusterMemory(nm1, rs, 0);

  rm1.stop();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse; //导入方法依赖的package包/类
@Test (timeout = 50000)
public void testReleasedContainerNotRecovered() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  rm1.start();

  RMApp app1 = rm1.submitApp(1024);
  final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  // Re-start RM
  conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 8000);
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
  am1.registerAppAttempt(true);

  // try to release a container before the container is actually recovered.
  final ContainerId runningContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  am1.allocate(null, Arrays.asList(runningContainer));

  // send container statuses to recover the containers
  List<NMContainerStatus> containerStatuses =
      createNMContainerStatusForApp(am1);
  nm1.registerNode(containerStatuses, null);

  // only the am container should be recovered.
  waitForNumContainersToRecover(1, rm2, am1.getApplicationAttemptId());

  final AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();
  // cached release request is cleaned.
  // assertFalse(scheduler.getPendingRelease().contains(runningContainer));

  AllocateResponse response = am1.allocate(null, null);
  // AM gets notified of the completed container.
  boolean receivedCompletedContainer = false;
  for (ContainerStatus status : response.getCompletedContainersStatuses()) {
    if (status.getContainerId().equals(runningContainer)) {
      receivedCompletedContainer = true;
    }
  }
  assertTrue(receivedCompletedContainer);

  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    public Boolean get() {
      // release cache is cleaned up and previous running container is not
      // recovered
      return scheduler.getApplicationAttempt(am1.getApplicationAttemptId())
        .getPendingRelease().isEmpty()
          && scheduler.getRMContainer(runningContainer) == null;
    }
  }, 1000, 20000);
}