Java ContainerLaunchContext.newInstance方法代碼示例

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
public boolean run() throws Exception {
  YarnClientApplication app = createApplication();
  ApplicationId appId = app.getNewApplicationResponse().getApplicationId();

  // Copy the application jar to the filesystem
  FileSystem fs = FileSystem.get(conf);
  String appIdStr = appId.toString();
  Path dstJarPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfJar), Constants.TF_JAR_NAME);
  Path dstLibPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfLib),
      Constants.TF_LIB_NAME);
  Map<String, Path> files = new HashMap<>();
  files.put(Constants.TF_JAR_NAME, dstJarPath);
  Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);
  Map<String, String> javaEnv = Utils.setJavaEnv(conf);
  String command = makeAppMasterCommand(dstLibPath.toString(), dstJarPath.toString());
  LOG.info("Make ApplicationMaster command: " + command);
  ContainerLaunchContext launchContext = ContainerLaunchContext.newInstance(
      localResources, javaEnv, Lists.newArrayList(command), null, null, null);
  Resource resource = Resource.newInstance(amMemory, amVCores);
  submitApplication(app, appName, launchContext, resource, amQueue);
  return awaitApplication(appId);
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
public ContainerLaunchContext buildContainerContext(Map<String, LocalResource> localResources, YacopConfig yacopConfig) {
ContainerLaunchContext ctx = null;
  try {
    List<String> commands = new ArrayList<>();
    //cmd
    Vector<CharSequence> vargs = new Vector<>(5);
    vargs.add("(" + yacopConfig.getCmd() + ")");
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
      command.append(str).append(" ");
    }
    commands.add(command.toString());
    //tokens
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    //ctx
    ctx = ContainerLaunchContext.newInstance(localResources, null, commands, null, allTokens.duplicate(), null);
    } catch (IOException e) {
      e.printStackTrace();
    }
    return ctx;
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
@Override
public void run() {
  try {
    Map<String, String> env = Utils.setJavaEnv(appMaster.getConfiguration());
    String current = ApplicationConstants.Environment.LD_LIBRARY_PATH.$$();
    env.put("LD_LIBRARY_PATH", current + ":" + "`pwd`");

    Map<String, Path> files = new HashMap<>();
    files.put(Constants.TF_JAR_NAME, new Path(tfJar));
    files.put(Constants.TF_LIB_NAME, new Path(tfLib));

    FileSystem fs = FileSystem.get(appMaster.getConfiguration());
    Map<String, LocalResource> localResources =
        Utils.makeLocalResources(fs, files);

    String command = makeContainerCommand(
        containerMemory, clusterSpec.toBase64EncodedJsonString(),
        taskInfo.jobName, taskInfo.taskIndex);

    LOG.info("Launching a new container."
        + ", containerId=" + container.getId()
        + ", containerNode=" + container.getNodeId().getHost()
        + ":" + container.getNodeId().getPort()
        + ", containerNodeURI=" + container.getNodeHttpAddress()
        + ", containerResourceMemory="
        + container.getResource().getMemorySize()
        + ", containerResourceVirtualCores="
        + container.getResource().getVirtualCores()
        + ", command: " + command);
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
        localResources, env, Lists.newArrayList(command), null, null, null, null);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
@Test(timeout=20000)
public void testAppSubmissionWithInvalidDelegationToken() throws Exception {
  Configuration conf = new Configuration();
  conf.set(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  MockRM rm = new MockRM(conf) {
    @Override
    protected void doSecureLogin() throws IOException {
      // Skip the login.
    }
  };
  ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes()); 
  ContainerLaunchContext amContainer =
      ContainerLaunchContext.newInstance(
          new HashMap<String, LocalResource>(), new HashMap<String, String>(),
          new ArrayList<String>(), new HashMap<String, ByteBuffer>(), tokens,
          new HashMap<ApplicationAccessType, String>());
  ApplicationSubmissionContext appSubContext =
      ApplicationSubmissionContext.newInstance(
          ApplicationId.newInstance(1234121, 0),
          "BOGUS", "default", Priority.UNDEFINED, amContainer, false,
          true, 1, Resource.newInstance(1024, 1), "BOGUS");
  SubmitApplicationRequest request =
      SubmitApplicationRequest.newInstance(appSubContext);
  try {
    rm.getClientRMService().submitApplication(request);
    fail("Error was excepted.");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Bad header found in token storage"));
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
@Override
public void launch(int n, List<String> command, List<String> hosts,
    boolean verbose) throws Exception {
  final List<String> chmod = new ArrayList<String>();
  chmod.add("chmod");
  chmod.add("a+rx");
  chmod.add(System.getenv(Environment.LOG_DIRS.name()));
  final ProcessBuilder pb = new ProcessBuilder(chmod);
  pb.redirectOutput(Redirect.INHERIT);
  pb.redirectError(Redirect.INHERIT);
  pb.start();
  redirect(command); // TODO clone before mutating the command
  rmClient.init(conf);
  rmClient.start();
  final NMClient nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  rmClient.registerApplicationMaster("", 0, "");
  for (int i = 0; i < n; ++i) {
    final ContainerRequest request = new ContainerRequest(
        Resource.newInstance(256, 1), null, null, Priority.newInstance(0));
    rmClient.addContainerRequest(request);
  }
  int responseId = 0;
  for (int containers = 0; containers < n;) {
    final AllocateResponse response = rmClient.allocate(responseId++);
    for (final Container container : response.getAllocatedContainers()) {
      final ContainerLaunchContext ctx = ContainerLaunchContext
          .newInstance(null, null, command, null, null, null);
      nmClient.startContainer(container, ctx);
    }
    containers += response.getAllocatedContainers().size();
    try {
      Thread.sleep(100);
    } catch (final InterruptedException e) {
    }
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
public ContainerLaunchContext buildContainerContext(Map<String, LocalResource> localResources, YacopConfig yacopConfig) {
  ContainerLaunchContext ctx = null;
  try {
    //env
    Map<String, String> env = new HashedMap();
    if (yacopConfig.getEngineType().equals("DOCKER")) {
      env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
      env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", yacopConfig.getEngineImage());
      if (yacopConfig.getVolumeConfigs() != null)
        env.put("YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS", getMountVolumePairList(yacopConfig));
      if (yacopConfig.getNetworkConfig() != null)
        env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", yacopConfig.getNetworkConfig().getName());
    }
    List<String> commands = new ArrayList<>();
    //cmd
    Vector<CharSequence> vargs = new Vector<>(5);
    vargs.add("(" + yacopConfig.getCmd() + ")");
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
      command.append(str).append(" ");
    }
    commands.add(command.toString());
    //tokens
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    //ctx
    ctx = ContainerLaunchContext.newInstance(
            localResources, env, commands, null, allTokens.duplicate(), null
    );
  } catch (IOException e) {
    e.printStackTrace();
  }
  return ctx;
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
static ContainerLaunchContext createContainerLaunchContext(
    Map<ApplicationAccessType, String> applicationACLs,
    Configuration conf, Token<JobTokenIdentifier> jobToken, Task remoteTask,
    final org.apache.hadoop.mapred.JobID oldJobId,
    WrappedJvmID jvmID,
    TaskAttemptListener taskAttemptListener,
    Credentials credentials) {

  synchronized (commonContainerSpecLock) {
    if (commonContainerSpec == null) {
      commonContainerSpec = createCommonContainerLaunchContext(
          applicationACLs, conf, jobToken, oldJobId, credentials);
    }
  }

  // Fill in the fields needed per-container that are missing in the common
  // spec.

  boolean userClassesTakesPrecedence =
    conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);

  // Setup environment by cloning from common env.
  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);
  if (userClassesTakesPrecedence) {
    myEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
  }
  MapReduceChildJVM.setVMEnv(myEnv, remoteTask);

  // Set up the launch command
  List<String> commands = MapReduceChildJVM.getVMCommand(
      taskAttemptListener.getAddress(), remoteTask, jvmID);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec
              .getServiceData().entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container = ContainerLaunchContext.newInstance(
      commonContainerSpec.getLocalResources(), myEnv, commands,
      myServiceData, commonContainerSpec.getTokens().duplicate(),
      applicationACLs);

  return container;
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
/**
 * Submits an application to Yarn,
 *
 * @param args
 *          the program arguments
 * @throws IOException
 *           if an error occurs
 * @throws YarnException
 *           if an error occurs
 */
public static void main(String[] args) throws IOException, YarnException {
  if (args.length <= 0) {
    System.err.println(
        "Usage: yarn " + Launcher.class.getCanonicalName() + " COMMAND");
    System.exit(1);
  }
  final List<String> command = new ArrayList<String>();
  command.add(args[0]);
  command.add("-D" + apgas.impl.Config.APGAS_LAUNCHER + "="
      + Launcher.class.getCanonicalName());
  command.add("-D" + apgas.impl.Config.APGAS_JAVA + "=" + args[0]);
  String classpath = "";
  for (int i = 1; i < args.length; i++) {
    if (args[i].equals("-cp") || args[i].equals("-classpath")) {
      classpath = args[++i];
    } else {
      command.add(args[i]);
    }
  }
  redirect(command);
  final Configuration conf = new YarnConfiguration();
  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  final String cp = String.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  final Map<String, String> env = Collections.singletonMap(
      Environment.CLASSPATH.name(),
      classpath + ApplicationConstants.CLASS_PATH_SEPARATOR + cp);
  final ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(null,
      env, command, null, null, null);
  final ApplicationSubmissionContext appContext = yarnClient
      .createApplication().getApplicationSubmissionContext();
  appContext.setAMContainerSpec(ctx);
  appContext.setResource(Resource.newInstance(256, 1));
  appContext.setMaxAppAttempts(1);
  final ApplicationId appId = appContext.getApplicationId();
  yarnClient.submitApplication(appContext);
  YarnApplicationState appState;
  for (;;) {
    appState = yarnClient.getApplicationReport(appId)
        .getYarnApplicationState();
    if (appState == YarnApplicationState.FINISHED
        || appState == YarnApplicationState.KILLED
        || appState == YarnApplicationState.FAILED) {
      break;
    }
    try {
      Thread.sleep(100);
    } catch (final InterruptedException e) {
    }
  }
  System.err.println(appId + " finished with state " + appState);
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //導入方法依賴的package包/類
public ApplicationId submitAppContext() throws YarnException, IOException, InterruptedException {

    yarnClient.start();

    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    appContext.setApplicationName(yacopConfig.getName());

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    FileSystem fs = FileSystem.get(conf);
    //upload the local docker image
    if (yacopConfig.isEngineLocalImage()) {
      boolean dockerImgUploaded = uploadDockerImage(fs, appId.toString(), yacopConfig.getEngineImage());
      if (dockerImgUploaded) {
        LOG.info("Local Docker image " + yacopConfig.getEngineImage() + " uploaded successfully");
      } else {
        LOG.info("Local Docker image " + yacopConfig.getEngineImage() + " upload failed, existing");
        System.exit(3);
      }
    }
    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null);
    configFile = serializeObj(appId, yacopConfig);
    addToLocalResources(fs, configFile, configFilePath, appId.toString(), localResources, null);

    Map<String, String> env = prepareEnv();
    List<String> commands = prepareCommands();

    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null);
    appContext.setAMContainerSpec(amContainer);
    Resource capability = Resource.newInstance(amMemory, amVCores);
    appContext.setResource(capability);

    // set security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
      Credentials credentials = new Credentials();
      String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
      if (tokenRenewer == null || tokenRenewer.length() == 0) {
        throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
      }

      // For now, only getting tokens for the default file-system.
      final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
      if (tokens != null) {
        for (Token<?> token : tokens) {
          LOG.info("Got dt for " + fs.getUri() + "; " + token);
        }
      }
      DataOutputBuffer dob = new DataOutputBuffer();
      credentials.writeTokenStorageToStream(dob);
      ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
      amContainer.setTokens(fsTokens);
    }

    Priority pri = Priority.newInstance(amPriority);
    appContext.setPriority(pri);
    appContext.setQueue(amQueue);

    yarnClient.submitApplication(appContext);
    return appId;
  }