Java YarnClient.submitApplication方法代碼示例

import org.apache.hadoop.yarn.client.api.YarnClient; //導入方法依賴的package包/類
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 

import org.apache.hadoop.yarn.client.api.YarnClient; //導入方法依賴的package包/類
public void run(String[] args) throws Exception {
	YarnConfiguration conf = new YarnConfiguration();
	YarnClient yarnClient = YarnClient.createYarnClient();
	yarnClient.init(conf);
	yarnClient.start();
	
	String cryptopath = "";

	final String zonepaths = args[0];
	final String exclasspath = args[1];
	final String globalKMS = args.length == 2 ? "localhost:16000" : args[2];
	final String pluginURI = conf.get(NuCypherExtRpcServer.NUCYPHER_EXT_NAMENODE_SERVICE_RPC_ADDRESS_KEY,
			InetSocketAddress.createUnresolved(new URI(conf.get("fs.defaultFS")).getHost(),
					NuCypherExtRpcServer.DEFAULT_PORT).toString());

	String providers =  conf.get(KEY_PROVIDER_PATH);
	
	YarnClientApplication app = yarnClient.createApplication();
	
	ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

	amContainer.setCommands(
			Collections.singletonList(
			//"/usr/bin/java"+
			Environment.JAVA_HOME.$$() + "/bin/java" + 
			" -Xmx1024M"+
			" ApplicationMasterKMS"+
			" " + zonepaths +
			" " + exclasspath +
			" " + globalKMS +
			" " + providers +
			" " + pluginURI +
			" 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + 
			" 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"));

	ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
	ApplicationId appId = appContext.getApplicationId();

	LocalResource appMasterJar = Records.newRecord(LocalResource.class);
	// setupAppMasterJar(jarPath, appMasterJar);
	Map<String, LocalResource> localResources = new HashMap<>();
	//localResources.put("prkeyrotation.jar", appMasterJar);
	setupExtJar(exclasspath, localResources, appId.toString());
	amContainer.setLocalResources(localResources);
	
	Map<String, String> appMasterEnv = new HashMap<String, String>();
	setupAppMasterEnv(appMasterEnv, exclasspath);
	amContainer.setEnvironment(appMasterEnv);
	
	Resource capability = Records.newRecord(Resource.class);
	capability.setMemory(1024);
	capability.setVirtualCores(1);
	
	appContext.setApplicationName("prkeyrotation");
	appContext.setAMContainerSpec(amContainer);
	appContext.setResource(capability);
	appContext.setQueue("default");
	
	System.out.println("Submitting application "+appId);
	yarnClient.submitApplication(appContext);
	
	ApplicationReport appReport = yarnClient.getApplicationReport(appId);
	YarnApplicationState appState = appReport.getYarnApplicationState();
	while (appState != YarnApplicationState.FINISHED &&
			appState != YarnApplicationState.KILLED &&
			appState != YarnApplicationState.FAILED) {
		Thread.sleep(100);
		appReport = yarnClient.getApplicationReport(appId);
		appState = appReport.getYarnApplicationState();
	}
	
	System.out.println("Application " + appId + " finished with " +
							" state " + appState +
							" at " + appReport.getFinishTime());
	
	
}
 

import org.apache.hadoop.yarn.client.api.YarnClient; //導入方法依賴的package包/類
@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

import org.apache.hadoop.yarn.client.api.YarnClient; //導入方法依賴的package包/類
private void run(String[] args) throws Exception {
    if (args.length == 0) {
        System.out.println("Usage: [options] [commands..]");
        System.out.println("options: [-file filename] [-appcp appClasspath]");
        return;
    }
    this.initArgs(args);
    // Create yarnClient
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
            .newRecord(ContainerLaunchContext.class);
    ApplicationSubmissionContext appContext = app
            .getApplicationSubmissionContext();
    // Submit application
    ApplicationId appId = appContext.getApplicationId();

    //add ctrl+c signal handler
    CtrlCHandler handler = new CtrlCHandler(appId, yarnClient);
    Signal intSignal = new Signal("INT");
    Signal.handle(intSignal, handler);

    // setup security token
    amContainer.setTokens(this.setupTokens());
    // setup cache-files and environment variables
    amContainer.setLocalResources(this.setupCacheFiles(appId));
    amContainer.setEnvironment(this.getEnvironment());
    String cmd = Environment.JAVA_HOME.$$() + "/bin/java"
            + " -Xmx900m"
            + " org.apache.hadoop.yarn.dmlc.ApplicationMaster"
            + this.cacheFileArg + ' ' + this.appArgs + " 1>"
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
            + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";

    LOG.debug(cmd);
    amContainer.setCommands(Collections.singletonList(cmd));

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(1024);
    capability.setVirtualCores(1);
    LOG.info("jobname=" + this.jobName + ",username=" + this.userName);

    appContext.setApplicationName(jobName + ":DMLC-YARN");
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue(queue);
    //appContext.setUser(userName);
    LOG.info("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED
            && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Application " + appId + " finished with"
            + " state " + appState + " at " + appReport.getFinishTime());
    if (!appReport.getFinalApplicationStatus().equals(
            FinalApplicationStatus.SUCCEEDED)) {
        System.err.println(appReport.getDiagnostics());
        System.out.println("Available queues:");
        for (QueueInfo q : yarnClient.getAllQueues()) {
          System.out.println(q.getQueueName());
        }

        yarnClient.killApplication(appId);
    }
}
 

import org.apache.hadoop.yarn.client.api.YarnClient; //導入方法依賴的package包/類
/**
 * Submits an application to Yarn,
 *
 * @param args
 *          the program arguments
 * @throws IOException
 *           if an error occurs
 * @throws YarnException
 *           if an error occurs
 */
public static void main(String[] args) throws IOException, YarnException {
  if (args.length <= 0) {
    System.err.println(
        "Usage: yarn " + Launcher.class.getCanonicalName() + " COMMAND");
    System.exit(1);
  }
  final List<String> command = new ArrayList<String>();
  command.add(args[0]);
  command.add("-D" + apgas.impl.Config.APGAS_LAUNCHER + "="
      + Launcher.class.getCanonicalName());
  command.add("-D" + apgas.impl.Config.APGAS_JAVA + "=" + args[0]);
  String classpath = "";
  for (int i = 1; i < args.length; i++) {
    if (args[i].equals("-cp") || args[i].equals("-classpath")) {
      classpath = args[++i];
    } else {
      command.add(args[i]);
    }
  }
  redirect(command);
  final Configuration conf = new YarnConfiguration();
  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  final String cp = String.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
      conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
          YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH));
  final Map<String, String> env = Collections.singletonMap(
      Environment.CLASSPATH.name(),
      classpath + ApplicationConstants.CLASS_PATH_SEPARATOR + cp);
  final ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(null,
      env, command, null, null, null);
  final ApplicationSubmissionContext appContext = yarnClient
      .createApplication().getApplicationSubmissionContext();
  appContext.setAMContainerSpec(ctx);
  appContext.setResource(Resource.newInstance(256, 1));
  appContext.setMaxAppAttempts(1);
  final ApplicationId appId = appContext.getApplicationId();
  yarnClient.submitApplication(appContext);
  YarnApplicationState appState;
  for (;;) {
    appState = yarnClient.getApplicationReport(appId)
        .getYarnApplicationState();
    if (appState == YarnApplicationState.FINISHED
        || appState == YarnApplicationState.KILLED
        || appState == YarnApplicationState.FAILED) {
      break;
    }
    try {
      Thread.sleep(100);
    } catch (final InterruptedException e) {
    }
  }
  System.err.println(appId + " finished with state " + appState);
}