Java BuilderUtils.newContainerTokenIdentifier方法代码示例

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
public MockContainer(ApplicationAttemptId appAttemptId,
    Dispatcher dispatcher, Configuration conf, String user,
    ApplicationId appId, int uniqId) throws IOException{

  this.user = user;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
      uniqId);
  this.launchContext = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  long currentTime = System.currentTimeMillis();
  this.containerTokenIdentifier =
      BuilderUtils.newContainerTokenIdentifier(BuilderUtils
        .newContainerToken(id, "127.0.0.1", 1234, user,
          BuilderUtils.newResource(1024, 1), currentTime + 10000, 123,
          "password".getBytes(), currentTime));
  this.state = ContainerState.NEW;
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
private LogAggregationContext getLogAggregationContextFromContainerToken(
    MockRM rm1, MockNM nm1, LogAggregationContext logAggregationContext)
    throws Exception {
  RMApp app2 = rm1.submitApp(200, logAggregationContext);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1.nodeHeartbeat(true);
  // request a container.
  am2.allocate("127.0.0.1", 512, 1, new ArrayList<ContainerId>());
  ContainerId containerId =
      ContainerId.newContainerId(am2.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId, RMContainerState.ALLOCATED);

  // acquire the container.
  List<Container> containers =
      am2.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(containers.get(0)
        .getContainerToken());
  return token.getLogAggregationContext();
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private void recoverContainer(RecoveredContainerState rcs)
    throws IOException {
  StartContainerRequest req = rcs.getStartRequest();
  ContainerLaunchContext launchContext = req.getContainerLaunchContext();
  ContainerTokenIdentifier token =
      BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
  ContainerId containerId = token.getContainerID();
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();

  LOG.info("Recovering " + containerId + " in state " + rcs.getStatus()
      + " with exit code " + rcs.getExitCode());

  if (context.getApplications().containsKey(appId)) {
    Credentials credentials = parseCredentials(launchContext);
    Container container = new ContainerImpl(getConfig(), dispatcher,
        context.getNMStateStore(), req.getContainerLaunchContext(),
        credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
        rcs.getDiagnostics(), rcs.getKilled());
    context.getContainers().put(containerId, container);
    dispatcher.getEventHandler().handle(
        new ApplicationContainerInitEvent(container));
  } else {
    if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
      LOG.warn(containerId + " has no corresponding application!");
    }
    LOG.info("Adding " + containerId + " to recently stopped containers");
    nodeStatusUpdater.addCompletedContainer(containerId);
  }
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
private static ContainerTokenIdentifier createContainerTokenId(
    ContainerId cid, NodeId nodeId, String user,
    NMContainerTokenSecretManager secretMgr) throws IOException {
  long rmid = cid.getApplicationAttemptId().getApplicationId()
      .getClusterTimestamp();
  ContainerTokenIdentifier ctid = new ContainerTokenIdentifier(cid,
      nodeId.toString(), user, BuilderUtils.newResource(1024, 1),
      System.currentTimeMillis() + 100000L,
      secretMgr.getCurrentKey().getKeyId(), rmid,
      Priority.newInstance(0), 0);
  Token token = BuilderUtils.newContainerToken(nodeId,
      secretMgr.createPassword(ctid), ctid);
  return BuilderUtils.newContainerTokenIdentifier(token);
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
@Test(timeout = 3000000)
public void testRMIdentifierOnContainerAllocation() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();

  // Register node1
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);

  // Submit an application
  RMApp app1 = rm.submitApp(2048);

  // kick the scheduling
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();

  am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
  AllocateResponse alloc1Response = am1.schedule(); // send the request

  // kick the scheduler
  nm1.nodeHeartbeat(true);
  while (alloc1Response.getAllocatedContainers().size() < 1) {
    LOG.info("Waiting for containers to be created for app 1...");
    sleep(1000);
    alloc1Response = am1.schedule();
  }

  // assert RMIdentifer is set properly in allocated containers
  Container allocatedContainer =
      alloc1Response.getAllocatedContainers().get(0);
  ContainerTokenIdentifier tokenId =
      BuilderUtils.newContainerTokenIdentifier(allocatedContainer
        .getContainerToken());
  Assert.assertEquals(MockRM.getClusterTimeStamp(), tokenId.getRMIdentifier());
  rm.stop();
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
@Override
public StartContainersResponse
    startContainers(StartContainersRequest requests)
        throws YarnException {
  StartContainerRequest request = requests.getStartContainerRequests().get(0);
  LOG.info("Container started by MyContainerManager: " + request);
  launched = true;
  Map<String, String> env =
      request.getContainerLaunchContext().getEnvironment();

  Token containerToken = request.getContainerToken();
  ContainerTokenIdentifier tokenId = null;

  try {
    tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
  } catch (IOException e) {
    throw RPCUtil.getRemoteException(e);
  }

  ContainerId containerId = tokenId.getContainerID();
  containerIdAtContainerManager = containerId.toString();
  attemptIdAtContainerManager =
      containerId.getApplicationAttemptId().toString();
  nmHostAtContainerManager = tokenId.getNmHostAddress();
  submitTimeAtContainerManager =
      Long.parseLong(env.get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
  maxAppAttempts =
      Integer.parseInt(env.get(ApplicationConstants.MAX_APP_ATTEMPTS_ENV));
  return StartContainersResponse.newInstance(
    new HashMap<String, ByteBuffer>(), new ArrayList<ContainerId>(),
    new HashMap<ContainerId, SerializedException>());
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
@Test(timeout = 90000)
public void testRemovePreviousCompletedContainersFromContext() throws Exception {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(
      NodeStatusUpdaterImpl
          .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
      "10000");
  nm.init(conf);
  NodeStatusUpdaterImpl nodeStatusUpdater =
      (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);

  ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
  Token containerToken =
      BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
          BuilderUtils.newResource(1024, 1), 0, 123,
          "password".getBytes(), 0);
  Container anyCompletedContainer = new ContainerImpl(conf, null,
      null, null, null, null,
      BuilderUtils.newContainerTokenIdentifier(containerToken)) {

    @Override
    public ContainerState getCurrentState() {
      return ContainerState.COMPLETE;
    }

    @Override
    public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
      return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.DONE;
    }
  };

  ContainerId runningContainerId =
      ContainerId.newContainerId(appAttemptId, 3);
  Token runningContainerToken =
      BuilderUtils.newContainerToken(runningContainerId, "anyHost",
        1234, "anyUser", BuilderUtils.newResource(1024, 1), 0, 123,
        "password".getBytes(), 0);
  Container runningContainer =
      new ContainerImpl(conf, null, null, null, null, null,
        BuilderUtils.newContainerTokenIdentifier(runningContainerToken)) {
        @Override
        public ContainerState getCurrentState() {
          return ContainerState.RUNNING;
        }

        @Override
        public org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState getContainerState() {
          return org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.RUNNING;
        }
      };

  nm.getNMContext().getApplications().putIfAbsent(appId,
      mock(Application.class));
  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
  nm.getNMContext().getContainers()
    .put(runningContainerId, runningContainer);

  Assert.assertEquals(2, nodeStatusUpdater.getContainerStatuses().size());

  List<ContainerId> ackedContainers = new ArrayList<ContainerId>();
  ackedContainers.add(cId);
  ackedContainers.add(runningContainerId);

  nodeStatusUpdater.removeOrTrackCompletedContainersFromContext(ackedContainers);

  Set<ContainerId> containerIdSet = new HashSet<ContainerId>();
  List<ContainerStatus> containerStatuses = nodeStatusUpdater.getContainerStatuses();
  for (ContainerStatus status : containerStatuses) {
    containerIdSet.add(status.getContainerId());
  }

  Assert.assertEquals(1, containerStatuses.size());
  // completed container is removed;
  Assert.assertFalse(containerIdSet.contains(cId));
  // running container is not removed;
  Assert.assertTrue(containerIdSet.contains(runningContainerId));
}
 

import org.apache.hadoop.yarn.server.utils.BuilderUtils; //导入方法依赖的package包/类
@Test
public void testCleanedupApplicationContainerCleanup() throws IOException {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(NodeStatusUpdaterImpl
          .YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS,
      "1000000");
  nm.init(conf);

  NodeStatusUpdaterImpl nodeStatusUpdater =
      (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);

  ContainerId cId = ContainerId.newContainerId(appAttemptId, 1);
  Token containerToken =
      BuilderUtils.newContainerToken(cId, "anyHost", 1234, "anyUser",
          BuilderUtils.newResource(1024, 1), 0, 123,
          "password".getBytes(), 0);
  Container anyCompletedContainer = new ContainerImpl(conf, null,
      null, null, null, null,
      BuilderUtils.newContainerTokenIdentifier(containerToken)) {

    @Override
    public ContainerState getCurrentState() {
      return ContainerState.COMPLETE;
    }
  };

  Application application = mock(Application.class);
  when(application.getApplicationState()).thenReturn(ApplicationState.RUNNING);
  nm.getNMContext().getApplications().putIfAbsent(appId, application);
  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);

  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());

  when(application.getApplicationState()).thenReturn(
      ApplicationState.FINISHING_CONTAINERS_WAIT);
  // The completed container will be saved in case of lost heartbeat.
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());

  nm.getNMContext().getContainers().put(cId, anyCompletedContainer);
  nm.getNMContext().getApplications().remove(appId);
  // The completed container will be saved in case of lost heartbeat.
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
  Assert.assertEquals(1, nodeStatusUpdater.getContainerStatuses().size());
}