Java NodeId.newInstance方法代码示例

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testNodeRegistrationSuccess() throws Exception {
  writeToHostsFile("host2");
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = NodeId.newInstance("host2", 1234);
  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  req.setNodeId(nodeId);
  req.setHttpPort(1234);
  req.setNMVersion(YarnVersionInfo.getVersion());
  // trying to register a invalid node.
  RegisterNodeManagerResponse response = resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response.getNodeAction());
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testAbsentNodeStatus() throws Exception {
  NodeId nodeId = NodeId.newInstance("Absenthost0", 0);
  NodeCLI cli = new NodeCLI();
  when(client.getNodeReports()).thenReturn(
              getNodeReports(0, NodeState.RUNNING));
  cli.setClient(client);
  cli.setSysOutPrintStream(sysOut);
  cli.setSysErrPrintStream(sysErr);
  int result = cli.run(new String[] { "-status", nodeId.toString() });
  assertEquals(0, result);
  verify(client).getNodeReports();
  verify(sysOut, times(1)).println(isA(String.class));
  verify(sysOut).println(
    "Could not find the node report for node id : " + nodeId.toString());
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test (timeout = 10000)
public void testAppsHelpCommand() throws Exception {
  ApplicationCLI cli = createAndGetAppCLI();
  ApplicationCLI spyCli = spy(cli);
  int result = spyCli.run(new String[] { "application", "-help" });
  Assert.assertTrue(result == 0);
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  result = cli.run(
      new String[] {"application", "-kill", applicationId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());

  sysOutStream.reset();
  NodeId nodeId = NodeId.newInstance("host0", 0);
  result = cli.run(
      new String[] { "application", "-status", nodeId.toString(), "args" });
  verify(spyCli).printUsage(any(String.class), any(Options.class));
  Assert.assertEquals(createApplicationCLIHelpMessage(),
      sysOutStream.toString());
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
private Container newContainerInstance(int id, Priority priority,
    Resource capability, String hostName) throws IOException {
  NodeId nodeId = NodeId.newInstance(hostName, 0);
  Container container = Records.newRecord(Container.class);
  container.setNodeId(nodeId);
  container.setPriority(priority);
  container.setResource(capability);
  container.setId(ContainerId.newContainerId(appAttemptId, ++containerId));
  Token token = Token.newInstance(nodeId.toString().getBytes(),
      nodeId.toString(), nodeId.toString().getBytes(), nodeId.toString());
  byte[] bytes = container.getId().toString().getBytes();
  ByteBuffer buffer = ByteBuffer.allocate(bytes.length);
  buffer.put(bytes);
  token.setIdentifier(buffer);
  container.setContainerToken(token);
  container.setNodeHttpAddress(hostName + ":0");
  return container;
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
public static NodeId toNodeId(String nodeIdStr) {
  String[] parts = nodeIdStr.split(":");
  if (parts.length != 2) {
    throw new IllegalArgumentException("Invalid NodeId [" + nodeIdStr
        + "]. Expected host:port");
  }
  try {
    NodeId nodeId =
        NodeId.newInstance(parts[0].trim(), Integer.parseInt(parts[1]));
    return nodeId;
  } catch (NumberFormatException e) {
    throw new IllegalArgumentException("Invalid port: " + parts[1], e);
  }
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
public NodeId toNodeId(String str) {
  if (str.contains(":")) {
    int idx = str.indexOf(':');
    NodeId id =
        NodeId.newInstance(str.substring(0, idx),
            Integer.valueOf(str.substring(idx + 1)));
    return id;
  } else {
    return NodeId.newInstance(str, CommonNodeLabelsManager.WILDCARD_PORT);
  }
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testNMTokenIdentifier() throws IOException {
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(1, 1), 1);
  NodeId nodeId = NodeId.newInstance("host0", 0);
  String applicationSubmitter = "usr0";
  int masterKeyId = 1;
  
  NMTokenIdentifier token = new NMTokenIdentifier(
      appAttemptId, nodeId, applicationSubmitter, masterKeyId);
  
  NMTokenIdentifier anotherToken = new NMTokenIdentifier();
  
  byte[] tokenContent = token.getBytes();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenContent, tokenContent.length);
  anotherToken.readFields(dib);
  
  // verify the whole record equals with original record
  Assert.assertEquals("Token is not the same after serialization " +
      "and deserialization.", token, anotherToken);
  
  // verify all properties are the same as original
  Assert.assertEquals(
      "appAttemptId from proto is not the same with original token",
      anotherToken.getApplicationAttemptId(), appAttemptId);
  
  Assert.assertEquals(
      "NodeId from proto is not the same with original token",
      anotherToken.getNodeId(), nodeId);
  
  Assert.assertEquals(
      "applicationSubmitter from proto is not the same with original token",
      anotherToken.getApplicationSubmitter(), applicationSubmitter);
  
  Assert.assertEquals(
      "masterKeyId from proto is not the same with original token",
      anotherToken.getKeyId(), masterKeyId);
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
private Container mockContainer(int i) {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  ApplicationAttemptId attemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(attemptId, i);
  nodeId = NodeId.newInstance("localhost", 0);
  // Create an empty record
  containerToken = recordFactory.newRecordInstance(Token.class);
  return Container.newInstance(containerId, nodeId, null, null, null,
    containerToken);
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void setup() throws IOException {
  super.setup();
  NodeId nodeId = NodeId.newInstance("0.0.0.0", 5555);
  ((NMContext)context).setNodeId(nodeId);
  dispatcher = createDispatcher();
  appEventHandler = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, appEventHandler);
  UserGroupInformation.setConfiguration(conf);
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testNodeStatusWithEmptyNodeLabels() throws Exception {
  NodeId nodeId = NodeId.newInstance("host0", 0);
  NodeCLI cli = new NodeCLI();
  when(client.getNodeReports()).thenReturn(
                  getNodeReports(3, NodeState.RUNNING));
  cli.setClient(client);
  cli.setSysOutPrintStream(sysOut);
  cli.setSysErrPrintStream(sysErr);
  int result = cli.run(new String[] { "-status", nodeId.toString() });
  assertEquals(0, result);
  verify(client).getNodeReports();
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  PrintWriter pw = new PrintWriter(baos);
  pw.println("Node Report : ");
  pw.println("\tNode-Id : host0:0");
  pw.println("\tRack : rack1");
  pw.println("\tNode-State : RUNNING");
  pw.println("\tNode-Http-Address : host1:8888");
  pw.println("\tLast-Health-Update : "
    + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
  pw.println("\tHealth-Report : ");
  pw.println("\tContainers : 0");
  pw.println("\tMemory-Used : 0MB");
  pw.println("\tMemory-Capacity : 0MB");
  pw.println("\tCPU-Used : 0 vcores");
  pw.println("\tCPU-Capacity : 0 vcores");
  pw.println("\tNode-Labels : ");
  pw.close();
  String nodeStatusStr = baos.toString("UTF-8");
  verify(sysOut, times(1)).println(isA(String.class));
  verify(sysOut).println(nodeStatusStr);
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testRunningToKilled() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(
      new RMAppAttemptEvent(
          applicationAttempt.getAppAttemptId(),
          RMAppAttemptEventType.KILL));

  // ignored ContainerFinished and Expire at FinalSaving if we were supposed
  // to Killed state.
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  applicationAttempt.handle(new RMAppAttemptEvent(
    applicationAttempt.getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState()); 
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  sendAttemptUpdateSavedEvent(applicationAttempt);
  assertEquals(RMAppAttemptState.KILLED,
      applicationAttempt.getAppAttemptState());
  assertEquals(0, applicationAttempt.getJustFinishedContainers().size());
  assertEquals(amContainer, applicationAttempt.getMasterContainer());
  assertEquals(0, application.getRanNodes().size());
  String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
      applicationAttempt.getAppAttemptId().getApplicationId());
  assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
  assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
  verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
  verifyAMHostAndPortInvalidated();
  verifyApplicationAttemptFinished(RMAppAttemptState.KILLED);
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testContainerCleanedWhileCommitting() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId =
    ApplicationAttemptId.newInstance(appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] {});

  AppContext appCtx = mock(AppContext.class);
  ClusterInfo clusterInfo = mock(ClusterInfo.class);
  Resource resource = mock(Resource.class);
  when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
  when(resource.getMemory()).thenReturn(1024);

  TaskAttemptImpl taImpl =
    new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
        splits, jobConf, taListener,
        new Token(), new Credentials(),
        new SystemClock(), appCtx);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");

  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_SCHEDULE));
  taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
      container, mock(Map.class)));
  taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_COMMIT_PENDING));

  assertEquals("Task attempt is not in commit pending state", taImpl.getState(),
      TaskAttemptState.COMMIT_PENDING);
  taImpl.handle(new TaskAttemptEvent(attemptId,
      TaskAttemptEventType.TA_CONTAINER_CLEANED));
  assertFalse("InternalError occurred trying to handle TA_CONTAINER_CLEANED",
      eventHandler.internalError);
  assertEquals("Task attempt is assigned locally", Locality.OFF_SWITCH,
      taImpl.getLocality());
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
public static NodeId newNodeID(String host, int port) {
  return NodeId.newInstance(host, port);
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
private static Container createMockContainer() {
  Container container = mock(Container.class);
  NodeId nodeId = NodeId.newInstance("foo.bar.org", 1234);
  when(container.getNodeId()).thenReturn(nodeId);
  return container;
}
 

import org.apache.hadoop.yarn.api.records.NodeId; //导入方法依赖的package包/类
@Test
public void testAppDiognosticEventOnNewTask() throws Exception {
  ApplicationId appId = ApplicationId.newInstance(1, 2);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  JobId jobId = MRBuilderUtils.newJobId(appId, 1);
  TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
  TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
  Path jobFile = mock(Path.class);

  MockEventHandler eventHandler = new MockEventHandler();
  TaskAttemptListener taListener = mock(TaskAttemptListener.class);
  when(taListener.getAddress()).thenReturn(
      new InetSocketAddress("localhost", 0));

  JobConf jobConf = new JobConf();
  jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
  jobConf.setBoolean("fs.file.impl.disable.cache", true);
  jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
  jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");

  TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
  when(splits.getLocations()).thenReturn(new String[] { "127.0.0.1" });

  AppContext appCtx = mock(AppContext.class);
  ClusterInfo clusterInfo = mock(ClusterInfo.class);
  Resource resource = mock(Resource.class);
  when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
  when(resource.getMemory()).thenReturn(1024);

  TaskAttemptImpl taImpl = new MapTaskAttemptImpl(taskId, 1, eventHandler,
      jobFile, 1, splits, jobConf, taListener,
      new Token(), new Credentials(), new SystemClock(), appCtx);

  NodeId nid = NodeId.newInstance("127.0.0.1", 0);
  ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
  Container container = mock(Container.class);
  when(container.getId()).thenReturn(contId);
  when(container.getNodeId()).thenReturn(nid);
  when(container.getNodeHttpAddress()).thenReturn("localhost:0");
  taImpl.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptId,
      "Task got killed"));
  assertFalse(
      "InternalError occurred trying to handle TA_DIAGNOSTICS_UPDATE on assigned task",
      eventHandler.internalError);
}