Java ContainerLaunchContext.setCommands方法代码示例

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private synchronized void launchDummyTask(Container container){
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    String new_command = "./launcher.py";
    String cmd = new_command + " 1>"
        + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
        + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
        + "/stderr";
    ctx.setCommands(Collections.singletonList(cmd));
    ctx.setTokens(setupTokens());
    ctx.setLocalResources(this.workerResources);
    synchronized (this){
        this.nmClient.startContainerAsync(container, ctx);
    }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
public static ContainerLaunchContext newContainerLaunchContext(
    Map<String, LocalResource> localResources,
    Map<String, String> environment, List<String> commands,
    Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
    Map<ApplicationAccessType, String> acls) {
  ContainerLaunchContext container = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  container.setLocalResources(localResources);
  container.setEnvironment(environment);
  container.setCommands(commands);
  container.setServiceData(serviceData);
  container.setTokens(tokens);
  container.setApplicationACLs(acls);
  return container;
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private ContainerLaunchContext createContainerContext() {
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

    amContainer.setLocalResources(setupEsYarnJar());
    amContainer.setEnvironment(setupEnv());
    amContainer.setCommands(setupCmd());

    return amContainer;
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private void launchContainer(Container container) {
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);

    ctx.setEnvironment(setupEnv(appConfig));
    ctx.setLocalResources(setupEsZipResource(appConfig));
    ctx.setCommands(setupEsScript(appConfig));

    log.info("About to launch container for command: " + ctx.getCommands());

    // setup container
    Map<String, ByteBuffer> startContainer = nmRpc.startContainer(container, ctx);
    log.info("Started container " + container);
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
@Test
public void testLogAggregationForRealContainerLaunch() throws IOException,
    InterruptedException, YarnException {

  this.containerManager.start();


  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  fileWriter.write("\necho Hello World! Stdout! > "
      + new File(localLogDir, "stdout"));
  fileWriter.write("\necho Hello World! Stderr! > "
      + new File(localLogDir, "stderr"));
  fileWriter.write("\necho Hello World! Syslog! > "
      + new File(localLogDir, "syslog"));
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  // ////// Construct the Container-id
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);

  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        TestContainerManager.createContainerToken(
          cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
          context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  this.containerManager.startContainers(allRequests);
  
  BaseContainerManagerTest.waitForContainerState(this.containerManager,
      cId, ContainerState.COMPLETE);

  this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays
      .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
  this.containerManager.stop();
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private void run(String[] args) throws Exception {
    if (args.length == 0) {
        System.out.println("Usage: [options] [commands..]");
        System.out.println("options: [-file filename] [-appcp appClasspath]");
        return;
    }
    this.initArgs(args);
    // Create yarnClient
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
            .newRecord(ContainerLaunchContext.class);
    ApplicationSubmissionContext appContext = app
            .getApplicationSubmissionContext();
    // Submit application
    ApplicationId appId = appContext.getApplicationId();

    //add ctrl+c signal handler
    CtrlCHandler handler = new CtrlCHandler(appId, yarnClient);
    Signal intSignal = new Signal("INT");
    Signal.handle(intSignal, handler);

    // setup security token
    amContainer.setTokens(this.setupTokens());
    // setup cache-files and environment variables
    amContainer.setLocalResources(this.setupCacheFiles(appId));
    amContainer.setEnvironment(this.getEnvironment());
    String cmd = Environment.JAVA_HOME.$$() + "/bin/java"
            + " -Xmx900m"
            + " org.apache.hadoop.yarn.dmlc.ApplicationMaster"
            + this.cacheFileArg + ' ' + this.appArgs + " 1>"
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
            + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";

    LOG.debug(cmd);
    amContainer.setCommands(Collections.singletonList(cmd));

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(1024);
    capability.setVirtualCores(1);
    LOG.info("jobname=" + this.jobName + ",username=" + this.userName);

    appContext.setApplicationName(jobName + ":DMLC-YARN");
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue(queue);
    //appContext.setUser(userName);
    LOG.info("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED
            && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Application " + appId + " finished with"
            + " state " + appState + " at " + appReport.getFinishTime());
    if (!appReport.getFinalApplicationStatus().equals(
            FinalApplicationStatus.SUCCEEDED)) {
        System.err.println(appReport.getDiagnostics());
        System.out.println("Available queues:");
        for (QueueInfo q : yarnClient.getAllQueues()) {
          System.out.println(q.getQueueName());
        }

        yarnClient.killApplication(appId);
    }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
private void testContainerLaunchAndSignal(SignalContainerCommand command)
    throws IOException, InterruptedException, YarnException {

  Signal signal = ContainerLaunch.translateCommandToSignal(command);
  containerManager.start();

  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  File processStartFile =
      new File(tmpDir, "start_file.txt").getAbsoluteFile();
  fileWriter.write("\numask 0"); // So that start file is readable by the test
  fileWriter.write("\necho Hello World! > " + processStartFile);
  fileWriter.write("\necho $$ >> " + processStartFile);
  fileWriter.write("\nexec sleep 1000s");
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources =
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
          containerLaunchContext,
          createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  int timeoutSecs = 0;
  while (!processStartFile.exists() && timeoutSecs++ < 20) {
    Thread.sleep(1000);
    LOG.info("Waiting for process start-file to be created");
  }
  Assert.assertTrue("ProcessStartFile doesn't exist!",
      processStartFile.exists());

  // Simulate NodeStatusUpdaterImpl sending CMgrSignalContainersEvent
  SignalContainerRequest signalReq =
      SignalContainerRequest.newInstance(cId, command);
  List<SignalContainerRequest> reqs = new ArrayList<SignalContainerRequest>();
  reqs.add(signalReq);
  containerManager.handle(new CMgrSignalContainersEvent(reqs));

  final ArgumentCaptor<ContainerSignalContext> signalContextCaptor =
      ArgumentCaptor.forClass(ContainerSignalContext.class);
  if (signal.equals(Signal.NULL)) {
    verify(exec, never()).signalContainer(signalContextCaptor.capture());
  } else {
    verify(exec, timeout(10000).atLeastOnce()).signalContainer(signalContextCaptor.capture());
    ContainerSignalContext signalContext = signalContextCaptor.getAllValues().get(0);
    Assert.assertEquals(cId, signalContext.getContainer().getContainerId());
    Assert.assertEquals(signal, signalContext.getSignal());
  }
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
public void startContainer()
    throws IOException, InterruptedException, YarnException {
  LOG.info("Start a container and wait until it is in RUNNING state");
  File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  if (Shell.WINDOWS) {
    fileWriter.println("@ping -n 100 127.0.0.1 >nul");
  } else {
    fileWriter.write("\numask 0");
    fileWriter.write("\nexec sleep 100");
  }
  fileWriter.close();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources =
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands =
      Arrays.asList(Shell.getRunScriptCommand(scriptFile));
  containerLaunchContext.setCommands(commands);
  Resource resource = Resource.newInstance(1024, 1);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
          containerLaunchContext,
          getContainerToken(resource));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  getContainerManager().startContainers(allRequests);
  // Make sure the container reaches RUNNING state
  ContainerId cId = TestContainerManager.createContainerId(0);
  BaseContainerManagerTest.waitForNMContainerState(
      getContainerManager(), cId,
      org.apache.hadoop.yarn.server.nodemanager.
          containermanager.container.ContainerState.RUNNING);
}
 

import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; //导入方法依赖的package包/类
protected ContainerLaunchContext setupApplicationMasterContainer(
		String yarnClusterEntrypoint,
		boolean hasLogback,
		boolean hasLog4j,
		boolean hasKrb5,
		int jobManagerMemoryMb) {
	// ------------------ Prepare Application Master Container  ------------------------------

	// respect custom JVM options in the YAML file
	String javaOpts = flinkConfiguration.getString(CoreOptions.FLINK_JVM_OPTIONS);
	if (flinkConfiguration.getString(CoreOptions.FLINK_JM_JVM_OPTIONS).length() > 0) {
		javaOpts += " " + flinkConfiguration.getString(CoreOptions.FLINK_JM_JVM_OPTIONS);
	}
	//applicable only for YarnMiniCluster secure test run
	//krb5.conf file will be available as local resource in JM/TM container
	if (hasKrb5) {
		javaOpts += " -Djava.security.krb5.conf=krb5.conf";
	}

	// Set up the container launch context for the application master
	ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

	final  Map<String, String> startCommandValues = new HashMap<>();
	startCommandValues.put("java", "$JAVA_HOME/bin/java");
	startCommandValues.put("jvmmem", "-Xmx" +
		Utils.calculateHeapSize(jobManagerMemoryMb, flinkConfiguration) +
		"m");
	startCommandValues.put("jvmopts", javaOpts);
	String logging = "";

	if (hasLogback || hasLog4j) {
		logging = "-Dlog.file=\"" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.log\"";

		if (hasLogback) {
			logging += " -Dlogback.configurationFile=file:" + CONFIG_FILE_LOGBACK_NAME;
		}

		if (hasLog4j) {
			logging += " -Dlog4j.configuration=file:" + CONFIG_FILE_LOG4J_NAME;
		}
	}

	startCommandValues.put("logging", logging);
	startCommandValues.put("class", yarnClusterEntrypoint);
	startCommandValues.put("redirects",
		"1> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.out " +
		"2> " + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/jobmanager.err");
	startCommandValues.put("args", "");

	final String commandTemplate = flinkConfiguration
		.getString(ConfigConstants.YARN_CONTAINER_START_COMMAND_TEMPLATE,
			ConfigConstants.DEFAULT_YARN_CONTAINER_START_COMMAND_TEMPLATE);
	final String amCommand =
		BootstrapTools.getStartCommand(commandTemplate, startCommandValues);

	amContainer.setCommands(Collections.singletonList(amCommand));

	LOG.debug("Application Master start command: " + amCommand);

	return amContainer;
}