Java YarnClientApplication類代碼示例

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
private static void mockYarnClient() throws YarnException, IOException {
  yarnClient = mock(YarnClient.class);
  doNothing().when(yarnClient).start();
  YarnClientApplication app = mock(YarnClientApplication.class);
  when(yarnClient.createApplication()).thenReturn(app);
  GetNewApplicationResponse appResponse = mock(GetNewApplicationResponse.class);
  when(app.getNewApplicationResponse()).thenReturn(appResponse);
  ApplicationSubmissionContext appContext = mock(ApplicationSubmissionContext.class);
  when(app.getApplicationSubmissionContext()).thenReturn(appContext);
  appId = mock(ApplicationId.class);
  when(appContext.getApplicationId()).thenReturn(appId);
  doNothing().when(appContext).setApplicationName(Matchers.anyString());
  report = mock(ApplicationReport.class);
  when(yarnClient.getApplicationReport(appId)).thenReturn(report);
  when(appId.getId()).thenReturn(1);
  when(appId.toString()).thenReturn("application_1465186316357_0001");
  when(report.getDiagnostics()).thenReturn("fake diagnostics");
  when(report.getQueue()).thenReturn("fake queue");
  when(report.getProgress()).thenReturn(0.5f);
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
public boolean run() throws Exception {
  YarnClientApplication app = createApplication();
  ApplicationId appId = app.getNewApplicationResponse().getApplicationId();

  // Copy the application jar to the filesystem
  FileSystem fs = FileSystem.get(conf);
  String appIdStr = appId.toString();
  Path dstJarPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfJar), Constants.TF_JAR_NAME);
  Path dstLibPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfLib),
      Constants.TF_LIB_NAME);
  Map<String, Path> files = new HashMap<>();
  files.put(Constants.TF_JAR_NAME, dstJarPath);
  Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);
  Map<String, String> javaEnv = Utils.setJavaEnv(conf);
  String command = makeAppMasterCommand(dstLibPath.toString(), dstJarPath.toString());
  LOG.info("Make ApplicationMaster command: " + command);
  ContainerLaunchContext launchContext = ContainerLaunchContext.newInstance(
      localResources, javaEnv, Lists.newArrayList(command), null, null, null);
  Resource resource = Resource.newInstance(amMemory, amVCores);
  submitApplication(app, appName, launchContext, resource, amQueue);
  return awaitApplication(appId);
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
public boolean run() throws YarnException, IOException {
	yarnClient.start();
	YarnClientApplication client = yarnClient.createApplication();
	GetNewApplicationResponse appResponse = client.getNewApplicationResponse();
	appId = appResponse.getApplicationId();
	LOG.info("Applicatoin ID = {}", appId);
	int maxMemory =	appResponse.getMaximumResourceCapability().getMemory();
	int maxVCores =	appResponse.getMaximumResourceCapability().getVirtualCores();
	LOG.info("Max memory = {} and max vcores = {}", maxMemory, maxVCores);
	YarnClusterMetrics clusterMetrics =
			yarnClient.getYarnClusterMetrics();
	LOG.info("Number of NodeManagers = {}", clusterMetrics.getNumNodeManagers());

	List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
	for (NodeReport node : nodeReports) {
		LOG.info("Node ID = {}, address = {}, containers = {}", node.getNodeId(), node.getHttpAddress(),
				node.getNumContainers());
	}
	List<QueueInfo> queueList = yarnClient.getAllQueues();
	for (QueueInfo queue : queueList) {
		LOG.info("Available queue: {} with capacity {} to {}", queue.getQueueName(), queue.getCapacity(),
				queue.getMaximumCapacity());
	}
	return true;
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
/**
 * If we do not have min/max, we may not be able to correctly request
 * the required resources from the RM for the app master
 * Memory ask has to be a multiple of min and less than max.
 * Dump out information about cluster capability as seen by the resource manager
 * @param app
 */
private void validateResourceForAM(YarnClientApplication app) {
  GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
  // TODO get min/max resource capabilities from RM and change memory ask if needed
  int maxMem = appResponse.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (amMemory > maxMem) {
    LOG.info("AM memory specified above max threshold of cluster. Using max value."
        + ", specified=" + amMemory
        + ", max=" + maxMem);
    amMemory = maxMem;
  }

  int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
  LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);

  if (amVCores > maxVCores) {
    LOG.info("AM virtual cores specified above max threshold of cluster. "
        + "Using max value." + ", specified=" + amVCores
        + ", max=" + maxVCores);
    amVCores = maxVCores;
  }
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
ApplicationId submitApplication(
    YarnClientApplication app,
    String appName,
    ContainerLaunchContext launchContext,
    Resource resource,
    String queue) throws Exception {
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  appContext.setApplicationName(appName);
  appContext.setApplicationTags(new HashSet<>());
  appContext.setAMContainerSpec(launchContext);
  appContext.setResource(resource);
  appContext.setQueue(queue);

  return yarnClient.submitApplication(appContext);
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
@Override
public YarnClientApplication createApplication()
    throws YarnException, IOException {
  ApplicationSubmissionContext context = Records.newRecord
      (ApplicationSubmissionContext.class);
  GetNewApplicationResponse newApp = getNewApplication();
  ApplicationId appId = newApp.getApplicationId();
  context.setApplicationId(appId);
  return new YarnClientApplication(newApp, context);
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
public YarnClientApplication newApp() {
    try {
        return client.createApplication();
    } catch (Exception ex) {
        throw new EsYarnException(ex);
    }
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
public ApplicationId run() {
    client.start();

    YarnClientApplication app = client.newApp();
    ApplicationSubmissionContext am = setupAM(app);
    ApplicationId appId = client.submitApp(am);
    return am.getApplicationId();
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
private ApplicationSubmissionContext setupAM(YarnClientApplication clientApp) {
    ApplicationSubmissionContext appContext = clientApp.getApplicationSubmissionContext();
    // already happens inside Hadoop but to be consistent
    appContext.setApplicationId(clientApp.getNewApplicationResponse().getApplicationId());
    appContext.setApplicationName(clientCfg.appName());
    appContext.setAMContainerSpec(createContainerContext());
    appContext.setResource(YarnCompat.resource(client.getConfiguration(), clientCfg.amMem(), clientCfg.amVCores()));
    appContext.setPriority(Priority.newInstance(clientCfg.amPriority()));
    appContext.setQueue(clientCfg.amQueue());
    appContext.setApplicationType(clientCfg.appType());
    YarnCompat.setApplicationTags(appContext, clientCfg.appTags());

    return appContext;
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
/**
 * Kills YARN application and stops YARN client.
 *
 * <p>Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient, YarnClientApplication yarnApplication) {
	LOG.info("Killing YARN application");

	try {
		yarnClient.killApplication(yarnApplication.getNewApplicationResponse().getApplicationId());
	} catch (Exception e) {
		// we only log a debug message here because the "killApplication" call is a best-effort
		// call (we don't know if the application has been deployed when the error occured).
		LOG.debug("Error while killing YARN application", e);
	}
	yarnClient.stop();
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 */
private ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
  YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
  ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
  ApplicationId applicationId = appSubmissionContext.getApplicationId();

  GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
  // Set up resource type requirements for ApplicationMaster
  Resource resource = prepareContainerResource(newApplicationResponse);

  // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
  Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

  ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
  amContainerLaunchContext.setLocalResources(appMasterLocalResources);
  amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
  amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(resource.getMemory())));
  if (UserGroupInformation.isSecurityEnabled()) {
    setupSecurityTokens(amContainerLaunchContext);
  }

  // Setup the application submission context
  appSubmissionContext.setApplicationName(this.applicationName);
  appSubmissionContext.setResource(resource);
  appSubmissionContext.setQueue(this.appQueueName);
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);

  // Also setup container local resources by copying local jars and files the container need to HDFS
  addContainerLocalResources(applicationId);

  // Submit the application
  LOGGER.info("Submitting application " + applicationId);
  this.yarnClient.submitApplication(appSubmissionContext);

  return applicationId;
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
@Override
protected void startUp() throws IOException {
  this.yarnClient = yarnClientFactory.connect();
  YarnClientApplication clientApp = getNewApplication();
  GetNewApplicationResponse newApp = clientApp.getNewApplicationResponse();
  ContainerLaunchContextFactory clcFactory = new ContainerLaunchContextFactory(newApp.getMaximumResourceCapability());
  
  ApplicationSubmissionContext appContext = clientApp.getApplicationSubmissionContext();
  this.applicationId = appContext.getApplicationId();
  appContext.setApplicationName(parameters.getApplicationName());
  
  // Setup the container for the application master.
  ContainerLaunchParameters appMasterParams = parameters.getApplicationMasterParameters(applicationId);
  ContainerLaunchContext clc = clcFactory.create(appMasterParams);

  appContext.setResource(clcFactory.createResource(appMasterParams));
  
  appContext.setAMContainerSpec(clc);
  appContext.setQueue(parameters.getQueue());
  appContext.setPriority(clcFactory.createPriority(appMasterParams.getPriority()));
  submitApplication(appContext);
  
  // Make sure we stop the application in the case that it isn't done already.
  Runtime.getRuntime().addShutdownHook(new Thread() {
    @Override
    public void run() {
      if (YarnClientServiceImpl.this.isRunning()) {
        YarnClientServiceImpl.this.stop();
      }
    }
  });
  
  stopwatch.start();
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
/**
 * Kills YARN application and stops YARN client.
 * <p>
 * Use this method to kill the App before it has been properly deployed
 */
private void failSessionDuringDeployment(YarnClient yarnClient,
        YarnClientApplication yarnApplication) {
  LOG.info("Killing YARN application");

  try {
    yarnClient.killApplication(yarnApplication.getNewApplicationResponse().
            getApplicationId());
  } catch (Exception e) {
    // we only log a debug message here because the "killApplication" call is a best-effort
    // call (we don't know if the application has been deployed when the error occured).
    LOG.debug("Error while killing YARN application", e);
  }
  yarnClient.stop();
}
 

import org.apache.hadoop.yarn.client.api.YarnClientApplication; //導入依賴的package包/類
@Override
public String getSeedQueryId() throws IOException {
  try {
    YarnClientApplication app = yarnClient.createApplication();
    return app.getApplicationSubmissionContext().getApplicationId().toString();
  } catch (Exception e) {
    LOG.error(e.getMessage(), e);

    throw new IOException(e.getMessage(), e);
  }
}