Java ContainerState.COMPLETE属性代码示例

private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
  List<NMContainerStatus> containerStatuses =
      new ArrayList<NMContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    if (!this.context.getApplications().containsKey(applicationId)) {
      context.getContainers().remove(containerId);
      continue;
    }
    NMContainerStatus status =
        container.getNMContainerStatus();
    containerStatuses.add(status);
    if (status.getContainerState() == ContainerState.COMPLETE) {
      // Adding to finished containers cache. Cache will keep it around at
      // least for #durationToTrackStoppedContainers duration. In the
      // subsequent call to stop container it will get removed from cache.
      addCompletedContainer(containerId);
    }
  }
  LOG.info("Sending out " + containerStatuses.size()
    + " NM container statuses: " + containerStatuses);
  return containerStatuses;
}
 

@Override
public void stopContainer(ContainerId containerId, NodeId nodeId)
    throws YarnException, IOException {
  StartedContainer startedContainer = getStartedContainer(containerId);

  // Only allow one request of stopping the container to move forward
  // When entering the block, check whether the precursor has already stopped
  // the container
  if (startedContainer != null) {
    synchronized (startedContainer) {
      if (startedContainer.state != ContainerState.RUNNING) {
        return;
      }
      stopContainerInternal(containerId, nodeId);
      // Only after successful
      startedContainer.state = ContainerState.COMPLETE;
      removeStartedContainer(startedContainer);
    }
  } else {
    stopContainerInternal(containerId, nodeId);
  }

}
 

@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
  List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
        container.cloneAndGetContainerStatus();
    if (containerStatus.getState() == ContainerState.COMPLETE) {
      if (isApplicationStopped(applicationId)) {
        if (LOG.isDebugEnabled()) {
          LOG.debug(applicationId + " is completing, " + " remove "
              + containerId + " from NM context.");
        }
        context.getContainers().remove(containerId);
        pendingCompletedContainers.put(containerId, containerStatus);
      } else {
        if (!isContainerRecentlyStopped(containerId)) {
          pendingCompletedContainers.put(containerId, containerStatus);
          // Adding to finished containers cache. Cache will keep it around at
          // least for #durationToTrackStoppedContainers duration. In the
          // subsequent call to stop container it will get removed from cache.
          addCompletedContainer(containerId);
        }
      }
    } else {
      containerStatuses.add(containerStatus);
    }
  }
  containerStatuses.addAll(pendingCompletedContainers.values());
  if (LOG.isDebugEnabled()) {
    LOG.debug("Sending out " + containerStatuses.size()
        + " container statuses: " + containerStatuses);
  }
  return containerStatuses;
}
 

/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 

public void cleanupContainersOnNMResync() {
  Map<ContainerId, Container> containers = context.getContainers();
  if (containers.isEmpty()) {
    return;
  }
  LOG.info("Containers still running on "
      + CMgrCompletedContainersEvent.Reason.ON_NODEMANAGER_RESYNC + " : "
      + containers.keySet());

  List<ContainerId> containerIds =
    new ArrayList<ContainerId>(containers.keySet());

  LOG.info("Waiting for containers to be killed");

  this.handle(new CMgrCompletedContainersEvent(containerIds,
    CMgrCompletedContainersEvent.Reason.ON_NODEMANAGER_RESYNC));

  /*
   * We will wait till all the containers change their state to COMPLETE. We
   * will not remove the container statuses from nm context because these
   * are used while re-registering node manager with resource manager.
   */
  boolean allContainersCompleted = false;
  while (!containers.isEmpty() && !allContainersCompleted) {
    allContainersCompleted = true;
    for (Entry<ContainerId, Container> container : containers.entrySet()) {
      if (((ContainerImpl) container.getValue()).getCurrentState()
          != ContainerState.COMPLETE) {
        allContainersCompleted = false;
        try {
          Thread.sleep(1000);
        } catch (InterruptedException ex) {
          LOG.warn("Interrupted while sleeping on container kill on resync",
            ex);
        }
        break;
      }
    }
  }
  // All containers killed
  if (allContainersCompleted) {
    LOG.info("All containers in DONE state");
  } else {
    LOG.info("Done waiting for containers to be killed. Still alive: " +
      containers.keySet());
  }
}
 

@Test(timeout = 90000)
public void testRemovePreviousCompletedContainersFromContext() throws Exception {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(
      NodeStatusUpdaterImpl
          .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
      "10000");
  nm.init(conf);
  NodeStatusUpdaterImpl nodeStatusUpdater =
      (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);

  ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
  Token containerToken =
      BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
          BuilderUtils.newResource(1024, 1), 0, 123,
          "password".getBytes(), 0);
  Container anyCompletedContainer = new ContainerImpl(conf, null,
      null, null, null, null,
      BuilderUtils.newContainerTokenIdentifier(containerToken)) {

    @Override
    public ContainerState getCurrentState() {
      return ContainerState.COMPLETE;
    }

    @Override
    public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
      return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE;
    }
  };

  ContainerId runningContainerId =
      ContainerId.newContainerId(appAttemptId, 3);
  Token runningContainerToken =
      BuilderUtils.newContainerToken(runningContainerId, "anyHost",
        1234, "anyUser", BuilderUtils.newResource(1024, 1), 0, 123,
        "password".getBytes(), 0);
  Container runningContainer =
      new ContainerImpl(conf, null, null, null, null, null,
        BuilderUtils.newContainerTokenIdentifier(runningContainerToken)) {
        @Override
        public ContainerState getCurrentState() {
          return ContainerState.RUNNING;
        }

        @Override
        public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
          return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING;
        }
      };

  nm.getNMContext().getApplications().putIfAbsent(appId,
      mock(Application.class));
  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
  nm.getNMContext().getContainers()
    .put(runningContainerId, runningContainer);

  Assert.assertEquals(2, nodeStatusUpdater.getContainerStatuses().size());

  List<ContainerId> ackedContainers = new ArrayList<ContainerId>();
  ackedContainers.add(cId);
  ackedContainers.add(runningContainerId);

  nodeStatusUpdater.removeOrTrackCompletedContainersFromContext(ackedContainers);

  Set<ContainerId> containerIdSet = new HashSet<ContainerId>();
  List<ContainerStatus> containerStatuses = nodeStatusUpdater.getContainerStatuses();
  for (ContainerStatus status : containerStatuses) {
    containerIdSet.add(status.getContainerId());
  }

  Assert.assertEquals(1, containerStatuses.size());
  // completed container is removed;
  Assert.assertFalse(containerIdSet.contains(cId));
  // running container is not removed;
  Assert.assertTrue(containerIdSet.contains(runningContainerId));
}
 

@Test
public void testCleanedupApplicationContainerCleanup() throws IOException {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(NodeStatusUpdaterImpl
          .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
      "1000000");
  nm.init(conf);

  NodeStatusUpdaterImpl nodeStatusUpdater =
      (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);

  ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
  Token containerToken =
      BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
          BuilderUtils.newResource(1024, 1), 0, 123,
          "password".getBytes(), 0);
  Container anyCompletedContainer = new ContainerImpl(conf, null,
      null, null, null, null,
      BuilderUtils.newContainerTokenIdentifier(containerToken)) {

    @Override
    public ContainerState getCurrentState() {
      return ContainerState.COMPLETE;
    }
  };

  Application application = mock(Application.class);
  when(application.getApplicationState()).thenReturn(ApplicationState.RUNNING);
  nm.getNMContext().getApplications().putIfAbsent(appId, application);
  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);

  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());

  when(application.getApplicationState()).thenReturn(
      ApplicationState.FINISHING_CONTAINERS_WAIT);
  // The completed container will be saved in case of lost heartbeat.
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());

  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
  nm.getNMContext().getApplications().remove(appId);
  // The completed container will be saved in case of lost heartbeat.
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
}