Java NodeHeartbeatResponse.getContainersToCleanup方法代码示例

import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; //导入方法依赖的package包/类
protected void waitForContainerCleanup(DrainDispatcher dispatcher, MockNM nm,
    NodeHeartbeatResponse resp) throws Exception {
  int waitCount = 0, cleanedConts = 0;
  List<ContainerId> contsToClean;
  do {
    dispatcher.await();
    contsToClean = resp.getContainersToCleanup();
    cleanedConts += contsToClean.size();
    if (cleanedConts >= 1) {
      break;
    }
    Thread.sleep(100);
    resp = nm.nodeHeartbeat(true);
  } while(waitCount++ < 200);

  if (contsToClean.isEmpty()) {
    LOG.error("Failed to get any containers to cleanup");
  } else {
    LOG.info("Got cleanup for " + contsToClean.get(0));
  }
  Assert.assertEquals(1, cleanedConts);
}
 

import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; //导入方法依赖的package包/类
private void testRMWritingMassiveHistory(MockRM rm) throws Exception {
  rm.start();
  MockNM nm = rm.registerNode("127.0.0.1:1234", 1024 * 10100);

  RMApp app = rm.submitApp(1024);
  nm.nodeHeartbeat(true);
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();

  int request = 10000;
  am.allocate("127.0.0.1", 1024, request, new ArrayList<ContainerId>());
  nm.nodeHeartbeat(true);
  List<Container> allocated =
      am.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  int waitCount = 0;
  int allocatedSize = allocated.size();
  while (allocatedSize < request && waitCount++ < 200) {
    Thread.sleep(300);
    allocated =
        am.allocate(new ArrayList<ResourceRequest>(),
          new ArrayList<ContainerId>()).getAllocatedContainers();
    allocatedSize += allocated.size();
    nm.nodeHeartbeat(true);
  }
  Assert.assertEquals(request, allocatedSize);

  am.unregisterAppAttempt();
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);

  NodeHeartbeatResponse resp = nm.nodeHeartbeat(true);
  List<ContainerId> cleaned = resp.getContainersToCleanup();
  int cleanedSize = cleaned.size();
  waitCount = 0;
  while (cleanedSize < allocatedSize && waitCount++ < 200) {
    Thread.sleep(300);
    resp = nm.nodeHeartbeat(true);
    cleaned = resp.getContainersToCleanup();
    cleanedSize += cleaned.size();
  }
  Assert.assertEquals(allocatedSize, cleanedSize);
  rm.waitForState(app.getApplicationId(), RMAppState.FINISHED);

  rm.stop();
}
 

import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; //导入方法依赖的package包/类
@SuppressWarnings("resource")
@Test
public void testAppCleanup() throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  MockRM rm = new MockRM();
  rm.start();

  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000);

  RMApp app = rm.submitApp(2000);

  //kick the scheduling
  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  
  //request for containers
  int request = 2;
  am.allocate("127.0.0.1" , 1000, request, 
      new ArrayList<ContainerId>());
  
  //kick the scheduler
  nm1.nodeHeartbeat(true);
  List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>()).getAllocatedContainers();
  int contReceived = conts.size();
  int waitCount = 0;
  while (contReceived < request && waitCount++ < 200) {
    LOG.info("Got " + contReceived + " containers. Waiting to get "
             + request);
    Thread.sleep(100);
    conts = am.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
    contReceived += conts.size();
    nm1.nodeHeartbeat(true);
  }
  Assert.assertEquals(request, contReceived);
  
  am.unregisterAppAttempt();
  NodeHeartbeatResponse resp = nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1,
      ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);

  //currently only containers are cleaned via this
  //AM container is cleaned via container launcher
  resp = nm1.nodeHeartbeat(true);
  List<ContainerId> containersToCleanup = resp.getContainersToCleanup();
  List<ApplicationId> appsToCleanup = resp.getApplicationsToCleanup();
  int numCleanedContainers = containersToCleanup.size();
  int numCleanedApps = appsToCleanup.size();
  waitCount = 0;
  while ((numCleanedContainers < 2 || numCleanedApps < 1)
      && waitCount++ < 200) {
    LOG.info("Waiting to get cleanup events.. cleanedConts: "
        + numCleanedContainers + " cleanedApps: " + numCleanedApps);
    Thread.sleep(100);
    resp = nm1.nodeHeartbeat(true);
    List<ContainerId> deltaContainersToCleanup =
        resp.getContainersToCleanup();
    List<ApplicationId> deltaAppsToCleanup = resp.getApplicationsToCleanup();
    // Add the deltas to the global list
    containersToCleanup.addAll(deltaContainersToCleanup);
    appsToCleanup.addAll(deltaAppsToCleanup);
    // Update counts now
    numCleanedContainers = containersToCleanup.size();
    numCleanedApps = appsToCleanup.size();
  }
  
  Assert.assertEquals(1, appsToCleanup.size());
  Assert.assertEquals(app.getApplicationId(), appsToCleanup.get(0));
  Assert.assertEquals(1, numCleanedApps);
  Assert.assertEquals(2, numCleanedContainers);

  rm.stop();
}
 

import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; //导入方法依赖的package包/类
@Override
public void middleStep() throws Exception {
  // we check the lifetime for each running containers
  ContainerSimulator cs = null;
  synchronized(completedContainerList) {
    while ((cs = containerQueue.poll()) != null) {
      runningContainers.remove(cs.getId());
      completedContainerList.add(cs.getId());
      LOG.debug(MessageFormat.format("Container {0} has completed",
              cs.getId()));
    }
  }
  
  // send heart beat
  NodeHeartbeatRequest beatRequest =
          Records.newRecord(NodeHeartbeatRequest.class);
  beatRequest.setLastKnownNMTokenMasterKey(masterKey);
  NodeStatus ns = Records.newRecord(NodeStatus.class);
  
  ns.setContainersStatuses(generateContainerStatusList());
  ns.setNodeId(node.getNodeID());
  ns.setKeepAliveApplications(new ArrayList<ApplicationId>());
  ns.setResponseId(RESPONSE_ID ++);
  ns.setNodeHealthStatus(NodeHealthStatus.newInstance(true, "", 0));
  beatRequest.setNodeStatus(ns);
  NodeHeartbeatResponse beatResponse =
      rm.getResourceTrackerService().nodeHeartbeat(beatRequest);
  if (! beatResponse.getContainersToCleanup().isEmpty()) {
    // remove from queue
    synchronized(releasedContainerList) {
      for (ContainerId containerId : beatResponse.getContainersToCleanup()){
        if (amContainerList.contains(containerId)) {
          // AM container (not killed?, only release)
          synchronized(amContainerList) {
            amContainerList.remove(containerId);
          }
          LOG.debug(MessageFormat.format("NodeManager {0} releases " +
              "an AM ({1}).", node.getNodeID(), containerId));
        } else {
          cs = runningContainers.remove(containerId);
          containerQueue.remove(cs);
          releasedContainerList.add(containerId);
          LOG.debug(MessageFormat.format("NodeManager {0} releases a " +
              "container ({1}).", node.getNodeID(), containerId));
        }
      }
    }
  }
  if (beatResponse.getNodeAction() == NodeAction.SHUTDOWN) {
    lastStep();
  }
}