Java ConverterUtils.toContainerId方法代码示例

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
private void sendErrorMessage() throws InterruptedException {
  List<TaskState> stateList = Arrays.asList(
      TaskState.CONTAINER_ALLOCATED, TaskState.CONTAINER_LAUNCHED, TaskState.CONTAINER_RUNNING);
  List<TaskStatus> list = statusManager.getTaskStatus(new HashSet<>(stateList));
  while (list.size() < taskNum) {
    Thread.sleep(2000);
    list = statusManager.getTaskStatus(new HashSet<>(stateList));
  }

  String containerIdStr = list.get(0).getContainerId();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);

  ContainerStatus status = Records.newRecord(ContainerStatus.class);
  status.setContainerId(containerId);
  status.setState(ContainerState.COMPLETE);
  status.setExitStatus(ExitStatusKey.SUCCEEDED.toInt());
  onContainersCompleted(Collections.singletonList(status));
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
private ContainerId verifyAndGetContainerId(Block html) {
  String containerIdStr = $(CONTAINER_ID);
  if (containerIdStr == null || containerIdStr.isEmpty()) {
    html.h1()._("Cannot get container logs without a ContainerId")._();
    return null;
  }
  ContainerId containerId = null;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
  } catch (IllegalArgumentException e) {
    html.h1()
        ._("Cannot get container logs for invalid containerId: "
            + containerIdStr)._();
    return null;
  }
  return containerId;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
public static FileInputStream openLogFileForRead(String containerIdStr, File logFile,
    Context context) throws IOException {
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  ApplicationId applicationId = containerId.getApplicationAttemptId()
      .getApplicationId();
  String user = context.getApplications().get(
      applicationId).getUser();
  
  try {
    return SecureIOUtils.openForRead(logFile, user, null);
  } catch (IOException e) {
    if (e.getMessage().contains(
      "did not match expected owner '" + user
          + "'")) {
      LOG.error(
          "Exception reading log file " + logFile.getAbsolutePath(), e);
      throw new IOException("Exception reading log file. Application submitted by '"
          + user
          + "' doesn't own requested log file : "
          + logFile.getName(), e);
    } else {
      throw new IOException("Exception reading log file. It might be because log "
          + "file was aggregated : " + logFile.getName(), e);
    }
  }
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@GET
@Path("/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainerInfo getNodeContainer(@PathParam("containerid") String id) {
  ContainerId containerId = null;
  init();
  try {
    containerId = ConverterUtils.toContainerId(id);
  } catch (Exception e) {
    throw new BadRequestException("invalid container id, " + id);
  }

  Container container = nmContext.getContainers().get(containerId);
  if (container == null) {
    throw new NotFoundException("container with id, " + id + ", not found");
  }
  return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri()
      .toString(), webapp.name());

}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
public List<ContainerId> getAllContainers(String applicationId,
    Configuration conf) {
  List<ContainerId> containers = new ArrayList<ContainerId>();
  String[] logDirs = HPCConfiguration.getHPCLogDirs(conf);
  for (String logDir : logDirs) {
    File appDir = new File(logDir, applicationId);
    if (appDir.exists()) {
      String[] containerDirs = appDir.list();
      if (containerDirs != null) {
        for (String containerDir : containerDirs) {
          try {
            ContainerId containerId = ConverterUtils
                .toContainerId(containerDir);
            containers.add(containerId);
          } catch (Exception e) {
            // Skip the non container dir's
            LOG.debug("Skipping non container dir : " + containerDir, e);
          }
        }
      }
    }
  }
  return containers;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
public static void initContainerList(List<Container> containerList, int length, Resource resource) {
  for (int i = 0; i < length; i++) {
    String containerIdStr = "container_" + System.currentTimeMillis() + "_0001_000001_" + (i + 2);
    ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
    NodeId nodeId = NodeId.newInstance(GlobalConstants.LOCAL_HOST_NAME, 3215);
    Container container = Container.newInstance(containerId,
        nodeId, GlobalConstants.LOCAL_HOST_NAME, resource, Priority.newInstance(1), null);
    containerList.add(container);
  }
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
protected static ContainerId parseContainerId(String containerId) {
  if (containerId == null || containerId.isEmpty()) {
    throw new NotFoundException("containerId, " + containerId
        + ", is empty or null");
  }
  ContainerId cid = ConverterUtils.toContainerId(containerId);
  if (cid == null) {
    throw new NotFoundException("containerId is null");
  }
  return cid;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
public void logs() {
  String containerIdStr = $(CONTAINER_ID);
  ContainerId containerId = null;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
  } catch (IllegalArgumentException e) {
    render(ContainerLogsPage.class);
    return;
  }
  ApplicationId appId =
      containerId.getApplicationAttemptId().getApplicationId();
  Application app = nmContext.getApplications().get(appId);
  if (app == null
      && nmConf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
          YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)) {
    String logServerUrl = nmConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
    String redirectUrl = null;
    if (logServerUrl == null || logServerUrl.isEmpty()) {
      redirectUrl = "false";
    } else {
      redirectUrl =
          url(logServerUrl, nmContext.getNodeId().toString(), containerIdStr,
              containerIdStr, $(APP_OWNER));
    }
    set(ContainerLogsPage.REDIRECT_URL, redirectUrl);
  }
  render(ContainerLogsPage.class);
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Override
protected void render(Block html) {
  ContainerId containerID;
  try {
    containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
  } catch (IllegalArgumentException e) {
    html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
    return;
  }

  DIV<Hamlet> div = html.div("#content");
  Container container = this.nmContext.getContainers().get(containerID);
  if (container == null) {
    div.h1("Unknown Container. Container might have completed, "
            + "please go back to the previous page and retry.")._();
    return;
  }
  ContainerInfo info = new ContainerInfo(this.nmContext, container);

  info("Container information")
    ._("ContainerID", info.getId())
    ._("ContainerState", info.getState())
    ._("ExitStatus", info.getExitStatus())
    ._("Diagnostics", info.getDiagnostics())
    ._("User", info.getUser())
    ._("TotalMemoryNeeded", info.getMemoryNeeded())
    ._("TotalVCoresNeeded", info.getVCoresNeeded())
    ._("logs", info.getShortLogLink(), "Link to logs");
  html._(InfoBlock.class);
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
private static void loadContainerToken(RecoveredContainerTokensState state,
    String key, String containerIdStr, byte[] value) throws IOException {
  ContainerId containerId;
  Long expTime;
  try {
    containerId = ConverterUtils.toContainerId(containerIdStr);
    expTime = Long.parseLong(asString(value));
  } catch (IllegalArgumentException e) {
    throw new IOException("Bad container token state for " + key, e);
  }
  state.activeTokens.put(containerId, expTime);
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterMidLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  //Create the file, but no end file so we should unregister with an error.
  fs.create(start).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is FAILED
  verifyFailedStatus((MRAppMasterTest)appMaster, "FAILED");
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterFailLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  Path end = MRApps.getEndJobCommitFailureFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  fs.create(start).close();
  fs.create(end).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.FAILED, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is FAILED
  verifyFailedStatus((MRAppMasterTest)appMaster, "FAILED");
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterMissingStaging() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);

  //Delete the staging directory
  File dir = new File(stagingDir);
  if(dir.exists()) {
    FileUtils.deleteDirectory(dir);
  }
  
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  //Copying the history file is disabled, but it is not really visible from 
  //here
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
public static void main(String[] args) {
  // get configuration from config file
  Configuration conf = new Configuration();
  conf.addResource(AngelConf.ANGEL_JOB_CONF_FILE);

  String containerIdStr = System.getenv(Environment.CONTAINER_ID.name());
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  ApplicationAttemptId applicationAttemptId = containerId.getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();
  String user = System.getenv(Environment.USER.name());

  // set localDir with enviroment set by nm.
  String[] localSysDirs =
      StringUtils.getTrimmedStrings(System.getenv(Environment.LOCAL_DIRS.name()));
  conf.setStrings(AngelConf.LOCAL_DIR, localSysDirs);
  LOG.info(AngelConf.LOCAL_DIR + " for child: " + conf.get(AngelConf.LOCAL_DIR));

  String psAgentindex = System.getenv(AngelEnvironment.PSAGENT_ID.name());
  String psAgentAttemptIndex = System.getenv(AngelEnvironment.PSAGENT_ATTEMPT_ID.name());
  String masterAddr = System.getenv(AngelEnvironment.LISTEN_ADDR.name());
  String portStr = System.getenv(AngelEnvironment.LISTEN_PORT.name());
  Location masterLocation = new Location(masterAddr, Integer.valueOf(portStr));

  LOG.info("psAgentindex=" + psAgentindex);
  LOG.info("psAgentAttemptIndex=" + psAgentAttemptIndex);
  LOG.info("masterLocation=" + masterLocation);
  LOG.info("user=" + user);
  LOG.info("appId=" + appId);

  PSAgentId psAgentId = new PSAgentId(Integer.valueOf(psAgentindex));
  PSAgentAttemptId psAgentAttemptId =
      new PSAgentAttemptId(psAgentId, Integer.valueOf(psAgentAttemptIndex));

  try {
    PSAgent psAgent =
        new PSAgent(conf, appId, user, psAgentAttemptId, masterAddr, Integer.valueOf(portStr),
            true, null);
    psAgent.initAndStart();
  } catch (Exception e) {
    LOG.fatal("Failed to start worker.", e);
  }
}