Java ApplicationSubmissionContext.setPriority方法代码示例

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
private ApplicationSubmissionContext setupAM(YarnClientApplication clientApp) {
    ApplicationSubmissionContext appContext = clientApp.getApplicationSubmissionContext();
    // already happens inside Hadoop but to be consistent
    appContext.setApplicationId(clientApp.getNewApplicationResponse().getApplicationId());
    appContext.setApplicationName(clientCfg.appName());
    appContext.setAMContainerSpec(createContainerContext());
    appContext.setResource(YarnCompat.resource(client.getConfiguration(), clientCfg.amMem(), clientCfg.amVCores()));
    appContext.setPriority(Priority.newInstance(clientCfg.amPriority()));
    appContext.setQueue(clientCfg.amQueue());
    appContext.setApplicationType(clientCfg.appType());
    YarnCompat.setApplicationTags(appContext, clientCfg.appTags());

    return appContext;
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
private RMAppImpl createAndPopulateNewRMApp(
    ApplicationSubmissionContext submissionContext, long submitTime,
    String user, boolean isRecovery) throws YarnException {
  // Do queue mapping
  if (!isRecovery) {
    if (rmContext.getQueuePlacementManager() != null) {
      // We only do queue mapping when it's a new application
      rmContext.getQueuePlacementManager().placeApplication(
          submissionContext, user);
    }
  }
  
  ApplicationId applicationId = submissionContext.getApplicationId();
  ResourceRequest amReq =
      validateAndCreateResourceRequest(submissionContext, isRecovery);

  // Verify and get the update application priority and set back to
  // submissionContext
  Priority appPriority = rmContext.getScheduler()
      .checkAndGetApplicationPriority(submissionContext.getPriority(), user,
          submissionContext.getQueue(), applicationId);
  submissionContext.setPriority(appPriority);

  // Create RMApp
  RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf,
      submissionContext.getApplicationName(), user,
      submissionContext.getQueue(), submissionContext, this.scheduler,
      this.masterService, submitTime, submissionContext.getApplicationType(),
      submissionContext.getApplicationTags(), amReq);

  // Concurrent app submissions with same applicationId will fail here
  // Concurrent app submissions with different applicationIds will not
  // influence each other
  if (rmContext.getRMApps().putIfAbsent(applicationId, application) !=
      null) {
    String message = "Application with id " + applicationId
        + " is already present! Cannot add a duplicate!";
    LOG.warn(message);
    throw new YarnException(message);
  }
  // Inform the ACLs Manager
  this.applicationACLsManager.addApplication(applicationId,
      submissionContext.getAMContainerSpec().getApplicationACLs());
  String appViewACLs = submissionContext.getAMContainerSpec()
      .getApplicationACLs().get(ApplicationAccessType.VIEW_APP);
  rmContext.getSystemMetricsPublisher().appACLsUpdated(
      application, appViewACLs, System.currentTimeMillis());
  return application;
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
 

import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; //导入方法依赖的package包/类
public ApplicationId submitAppContext() throws YarnException, IOException, InterruptedException {

    yarnClient.start();

    YarnClientApplication app = yarnClient.createApplication();
    GetNewApplicationResponse appResponse = app.getNewApplicationResponse();

    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    appContext.setApplicationName(yacopConfig.getName());

    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    FileSystem fs = FileSystem.get(conf);
    //upload the local docker image
    if (yacopConfig.isEngineLocalImage()) {
      boolean dockerImgUploaded = uploadDockerImage(fs, appId.toString(), yacopConfig.getEngineImage());
      if (dockerImgUploaded) {
        LOG.info("Local Docker image " + yacopConfig.getEngineImage() + " uploaded successfully");
      } else {
        LOG.info("Local Docker image " + yacopConfig.getEngineImage() + " upload failed, existing");
        System.exit(3);
      }
    }
    addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), localResources, null);
    configFile = serializeObj(appId, yacopConfig);
    addToLocalResources(fs, configFile, configFilePath, appId.toString(), localResources, null);

    Map<String, String> env = prepareEnv();
    List<String> commands = prepareCommands();

    ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(localResources, env, commands, null, null, null);
    appContext.setAMContainerSpec(amContainer);
    Resource capability = Resource.newInstance(amMemory, amVCores);
    appContext.setResource(capability);

    // set security tokens
    if (UserGroupInformation.isSecurityEnabled()) {
      Credentials credentials = new Credentials();
      String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
      if (tokenRenewer == null || tokenRenewer.length() == 0) {
        throw new IOException("Can't get Master Kerberos principal for the RM to use as renewer");
      }

      // For now, only getting tokens for the default file-system.
      final Token<?> tokens[] = fs.addDelegationTokens(tokenRenewer, credentials);
      if (tokens != null) {
        for (Token<?> token : tokens) {
          LOG.info("Got dt for " + fs.getUri() + "; " + token);
        }
      }
      DataOutputBuffer dob = new DataOutputBuffer();
      credentials.writeTokenStorageToStream(dob);
      ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
      amContainer.setTokens(fsTokens);
    }

    Priority pri = Priority.newInstance(amPriority);
    appContext.setPriority(pri);
    appContext.setQueue(amQueue);

    yarnClient.submitApplication(appContext);
    return appId;
  }