Java ContainerId類代碼示例

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Override
public void handle(ContainerAllocatorEvent event) {
  ContainerId cId =
      ContainerId.newContainerId(getContext().getApplicationAttemptId(),
        containerCount++);
  NodeId nodeId = NodeId.newInstance(NM_HOST, NM_PORT);
  Resource resource = Resource.newInstance(1234, 2, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(cId, nodeId.toString(), "user",
      resource, System.currentTimeMillis() + 10000, 42, 42,
      Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);
  Container container = Container.newInstance(cId, nodeId,
      NM_HOST + ":" + NM_HTTP_PORT, resource, null, containerToken);
  JobID id = TypeConverter.fromYarn(applicationId);
  JobId jobId = TypeConverter.toYarn(id);
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.REDUCE,
      100)));
  getContext().getEventHandler().handle(new JobHistoryEvent(jobId, 
      new NormalizedResourceEvent(
          org.apache.hadoop.mapreduce.TaskType.MAP,
      100)));
  getContext().getEventHandler().handle(
      new TaskAttemptContainerAssignedEvent(event.getAttemptID(),
          container, null));
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Override
public ApplicationState transition(ApplicationImpl app,
    ApplicationEvent event) {
  ApplicationFinishEvent appEvent = (ApplicationFinishEvent)event;
  if (app.containers.isEmpty()) {
    // No container to cleanup. Cleanup app level resources.
    app.handleAppFinishWithContainersCleanedup();
    return ApplicationState.APPLICATION_RESOURCES_CLEANINGUP;
  }

  // Send event to ContainersLauncher to finish all the containers of this
  // application.
  for (ContainerId containerID : app.containers.keySet()) {
    app.dispatcher.getEventHandler().handle(
        new ContainerKillEvent(containerID,
            ContainerExitStatus.KILLED_AFTER_APP_COMPLETION,
            "Container killed on application-finish event: " + appEvent.getDiagnostic()));
  }
  return ApplicationState.FINISHING_CONTAINERS_WAIT;
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
/**
 * This method produces an Allocation that includes the current view
 * of the resources that will be allocated to and preempted from this
 * application.
 *
 * @param rc
 * @param clusterResource
 * @param minimumAllocation
 * @return an allocation
 */
public synchronized Allocation getAllocation(ResourceCalculator rc,
    Resource clusterResource, Resource minimumAllocation) {

  Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
      new HashSet<ContainerId>(containersToPreempt));
  containersToPreempt.clear();
  Resource tot = Resource.newInstance(0, 0, 0);
  for(ContainerId c : currentContPreemption){
    Resources.addTo(tot,
        liveContainers.get(c).getContainer().getResource());
  }
  int numCont = (int) Math.ceil(
      Resources.divide(rc, clusterResource, tot, minimumAllocation));
  ResourceRequest rr = ResourceRequest.newInstance(
      Priority.UNDEFINED, ResourceRequest.ANY,
      minimumAllocation, numCont);
  ContainersAndNMTokensAllocation allocation =
      pullNewlyAllocatedContainersAndNMTokens();
  Resource headroom = getHeadroom();
  setApplicationHeadroomForMetrics(headroom);
  return new Allocation(allocation.getContainerList(), headroom, null,
    currentContPreemption, Collections.singletonList(rr),
    allocation.getNMTokenList());
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
/**
 * A helper api for creating an audit log for a successful event.
 */
static String createSuccessLog(String user, String operation, String target,
    ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) {
  StringBuilder b = new StringBuilder();
  start(Keys.USER, user, b);
  addRemoteIP(b);
  add(Keys.OPERATION, operation, b);
  add(Keys.TARGET, target ,b);
  add(Keys.RESULT, AuditConstants.SUCCESS, b);
  if (appId != null) {
    add(Keys.APPID, appId.toString(), b);
  }
  if (attemptId != null) {
    add(Keys.APPATTEMPTID, attemptId.toString(), b);
  }
  if (containerId != null) {
    add(Keys.CONTAINERID, containerId.toString(), b);
  }
  return b.toString();
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 2)));
  client.stop();
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
boolean remove(TaskAttemptId tId) {
  ContainerId containerId = null;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    containerId = maps.remove(tId).getId();
  } else {
    containerId = reduces.remove(tId).getId();
    if (containerId != null) {
      boolean preempted = preemptionWaitingReduces.remove(tId);
      if (preempted) {
        LOG.info("Reduce preemption successful " + tId);
      }
    }
  }
  
  if (containerId != null) {
    containerToAttemptMap.remove(containerId);
    return true;
  }
  return false;
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
protected void releaseContainers(List<ContainerId> containers,
    SchedulerApplicationAttempt attempt) {
  for (ContainerId containerId : containers) {
    RMContainer rmContainer = getRMContainer(containerId);
    if (rmContainer == null) {
      if (System.currentTimeMillis() - ResourceManager.getClusterTimeStamp()
          < nmExpireInterval) {
        LOG.info(containerId + " doesn't exist. Add the container"
            + " to the release request cache as it maybe on recovery.");
        synchronized (attempt) {
          attempt.getPendingRelease().add(containerId);
        }
      } else {
        RMAuditLogger.logFailure(attempt.getUser(),
          AuditConstants.RELEASE_CONTAINER,
          "Unauthorized access or invalid container", "Scheduler",
          "Trying to release container not owned by app or with invalid id.",
          attempt.getApplicationId(), containerId);
      }
    }
    completedContainer(rmContainer,
      SchedulerUtils.createAbnormalContainerStatus(containerId,
        SchedulerUtils.RELEASED_CONTAINER), RMContainerEventType.RELEASED);
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
protected void createReleaseCache() {
  // Cleanup the cache after nm expire interval.
  new Timer().schedule(new TimerTask() {
    @Override
    public void run() {
      for (SchedulerApplication<T> app : applications.values()) {

        T attempt = app.getCurrentAppAttempt();
        synchronized (attempt) {
          for (ContainerId containerId : attempt.getPendingRelease()) {
            RMAuditLogger.logFailure(
              app.getUser(),
              AuditConstants.RELEASE_CONTAINER,
              "Unauthorized access or invalid container",
              "Scheduler",
              "Trying to release container not owned by app or with invalid id.",
              attempt.getApplicationId(), containerId);
          }
          attempt.getPendingRelease().clear();
        }
      }
      LOG.info("Release request cache is cleaned up");
    }
  }, nmExpireInterval);
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Override
public void handle(ContainerEvent event) {
  try {
    this.writeLock.lock();

    ContainerId containerID = event.getContainerID();
    LOG.debug("Processing " + containerID + " of type " + event.getType());

    ContainerState oldState = stateMachine.getCurrentState();
    ContainerState newState = null;
    try {
      newState =
          stateMachine.doTransition(event.getType(), event);
    } catch (InvalidStateTransitonException e) {
      LOG.warn("Can't handle this event at current state: Current: ["
          + oldState + "], eventType: [" + event.getType() + "]", e);
    }
    if (oldState != newState) {
      LOG.info("Container " + containerID + " transitioned from "
          + oldState
          + " to " + newState);
    }
  } finally {
    this.writeLock.unlock();
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
ApplicationHistoryClientService mockApplicationHistoryClientService(int numApps,
    int numAppAttempts, int numContainers) throws Exception {
  ApplicationHistoryManager ahManager =
      new MockApplicationHistoryManagerImpl(store);
  ApplicationHistoryClientService historyClientService =
      new ApplicationHistoryClientService(ahManager);
  for (int i = 1; i <= numApps; ++i) {
    ApplicationId appId = ApplicationId.newInstance(0, i);
    writeApplicationStartData(appId);
    for (int j = 1; j <= numAppAttempts; ++j) {
      ApplicationAttemptId appAttemptId =
          ApplicationAttemptId.newInstance(appId, j);
      writeApplicationAttemptStartData(appAttemptId);
      for (int k = 1; k <= numContainers; ++k) {
        ContainerId containerId = ContainerId.newContainerId(appAttemptId, k);
        writeContainerStartData(containerId);
        writeContainerFinishData(containerId);
      }
      writeApplicationAttemptFinishData(appAttemptId);
    }
    writeApplicationFinishData(appId);
  }
  return historyClientService;
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Override
public void transition(LocalizedResource rsrc, ResourceEvent event) {
  ResourceLocalizedEvent locEvent = (ResourceLocalizedEvent) event;
  rsrc.localPath =
      Path.getPathWithoutSchemeAndAuthority(locEvent.getLocation());
  rsrc.size = locEvent.getSize();
  for (ContainerId container : rsrc.ref) {
    rsrc.dispatcher.getEventHandler().handle(
        new ContainerResourceLocalizedEvent(
          container, rsrc.rsrc, rsrc.localPath));
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
/**
 * Finds the local directories that logs for the given container are stored
 * on.
 */
public static List<File> getContainerLogDirs(ContainerId containerId,
    String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);

  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  // It is not required to have null check for container ( container == null )
  // and throw back exception.Because when container is completed, NodeManager
  // remove container information from its NMContext.Configuring log
  // aggregation to false, container log view request is forwarded to NM. NM
  // does not have completed container information,but still NM serve request for
  // reading container logs. 
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  return getContainerLogDirs(containerId, context.getLocalDirsHandler());
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
public NMContainerTokenSecretManager(Configuration conf,
    NMStateStoreService stateStore) {
  super(conf);
  recentlyStartedContainerTracker =
      new TreeMap<Long, List<ContainerId>>();
  this.stateStore = stateStore;
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Test
public void testContainerPage() throws Exception {
  Injector injector =
      WebAppTests.createMockInjector(ApplicationBaseProtocol.class,
        mockApplicationHistoryClientService(1, 1, 1));
  ContainerPage containerPageInstance =
      injector.getInstance(ContainerPage.class);

  containerPageInstance.render();
  WebAppTests.flushOutput(injector);

  containerPageInstance.set(
    YarnWebParams.CONTAINER_ID,
    ContainerId
      .newContainerId(
        ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1),
        1).toString());
  containerPageInstance.render();
  WebAppTests.flushOutput(injector);
}
 

import org.apache.hadoop.yarn.api.records.ContainerId; //導入依賴的package包/類
@Test
public void testMassiveWriteContainerHistory() throws IOException {
  long mb = 1024 * 1024;
  Runtime runtime = Runtime.getRuntime();
  long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
  int numContainers = 100000;
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  for (int i = 1; i <= numContainers; ++i) {
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, i);
    writeContainerStartData(containerId);
    writeContainerFinishData(containerId);
  }
  long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
  Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400);
}