Java YarnApplicationState.FINISHED屬性代碼示例

public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 

/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}
 

private void killUnFinishedApplication(ApplicationId appId) throws IOException {
	ApplicationReport application = null;
	try {
		application = resMgrDelegate.getApplicationReport(appId);
	} catch (YarnException e) {
		throw new IOException(e);
	}
	if (application.getYarnApplicationState() == YarnApplicationState.FINISHED
			|| application.getYarnApplicationState() == YarnApplicationState.FAILED
			|| application.getYarnApplicationState() == YarnApplicationState.KILLED) {
		return;
	}
	killApplication(appId);
}
 

/**
 * Moves the application with the given ID to the given queue.
 */
private void moveApplicationAcrossQueues(String applicationId, String queue)
    throws YarnException, IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport appReport = client.getApplicationReport(appId);
  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Moving application " + applicationId + " to queue " + queue);
    client.moveApplicationAcrossQueues(appId, queue);
    sysout.println("Successfully completed move.");
  }
}
 

private void killUnFinishedApplication(ApplicationId appId)
    throws IOException {
  ApplicationReport application = null;
  try {
    application = resMgrDelegate.getApplicationReport(appId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
  if (application.getYarnApplicationState() == YarnApplicationState.FINISHED
      || application.getYarnApplicationState() == YarnApplicationState.FAILED
      || application.getYarnApplicationState() == YarnApplicationState.KILLED) {
    return;
  }
  killApplication(appId);
}
 

/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. 
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
    throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in 
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
        + ", appId=" + appId.getId()
        + ", clientToAMToken=" + report.getClientToAMToken()
        + ", appDiagnostics=" + report.getDiagnostics()
        + ", appMasterHost=" + report.getHost()
        + ", appQueue=" + report.getQueue()
        + ", appMasterRpcPort=" + report.getRpcPort()
        + ", appStartTime=" + report.getStartTime()
        + ", yarnAppState=" + report.getYarnApplicationState().toString()
        + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
        + ", appTrackingUrl=" + report.getTrackingUrl()
        + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;        
      }
      else {
        LOG.info("Application did finished unsuccessfully."
            + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
            + ". Breaking monitoring loop");
        return false;
      }			  
    }
    else if (YarnApplicationState.KILLED == state	
        || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
          + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
          + ". Breaking monitoring loop");
      return false;
    }			

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;				
    }
  }			

}
 

public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
 

public List<ApplicationReport> getApplications(int id) {
  String result = null;
  String squeueCmd = conf.get(
      HPCConfiguration.YARN_APPLICATION_HPC_COMMAND_SLURM_SQUEUE,
      HPCConfiguration.DEFAULT_YARN_APPLICATION_HPC_COMMAND_SLURM_SQUEUE);
  List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
  try {
    String argJ = id < 0 ? "" : "-j " + id;
    result = Shell.execCommand(squeueCmd, "-h", argJ, "-o %i %P %j %u %k");

    String[] resultLines = result.split("\\n");
    for (String line : resultLines) {
      if (line == null || line.length() == 0) {
        return applications;
      }

      String[] lineParts = line.trim().split("\\s+");

      if (!("YarnMaster".equals(lineParts[2]) && StringUtils
          .isNotEmpty(lineParts[4])))
        continue;

      String[] addr = lineParts[4].split(":");
      if (addr.length != 3)
        continue;

      ApplicationReport appReport = Records
          .newRecord(ApplicationReport.class);

      int jobid = Integer.parseInt(lineParts[0]);
      ApplicationId applicationId = ApplicationId.newInstance(
          getClusterTimestamp(), jobid);

      appReport.setApplicationId(applicationId);
      appReport.setTrackingUrl("");
      appReport.setQueue(lineParts[1]);
      appReport.setName(lineParts[2]);
      appReport.setUser(lineParts[3]);

      appReport.setHost(addr[1]);
      appReport.setRpcPort(Integer.parseInt(addr[2]));

      String stateStr = addr[0].toUpperCase();
      YarnApplicationState state = YarnApplicationState.valueOf(stateStr);
      appReport.setYarnApplicationState(state);
      if (state == YarnApplicationState.FINISHED) {
        appReport.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
      }else{
        appReport.setFinalApplicationStatus(FinalApplicationStatus.UNDEFINED);
      }
      applications.add(appReport);
    }
  } catch (Throwable t) {
    throw new YarnRuntimeException(result, t);
  }
  return applications;
}
 

private void run(String[] args) throws Exception {
    if (args.length == 0) {
        System.out.println("Usage: [options] [commands..]");
        System.out.println("options: [-file filename] [-appcp appClasspath]");
        return;
    }
    this.initArgs(args);
    // Create yarnClient
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
            .newRecord(ContainerLaunchContext.class);
    ApplicationSubmissionContext appContext = app
            .getApplicationSubmissionContext();
    // Submit application
    ApplicationId appId = appContext.getApplicationId();

    //add ctrl+c signal handler
    CtrlCHandler handler = new CtrlCHandler(appId, yarnClient);
    Signal intSignal = new Signal("INT");
    Signal.handle(intSignal, handler);

    // setup security token
    amContainer.setTokens(this.setupTokens());
    // setup cache-files and environment variables
    amContainer.setLocalResources(this.setupCacheFiles(appId));
    amContainer.setEnvironment(this.getEnvironment());
    String cmd = Environment.JAVA_HOME.$$() + "/bin/java"
            + " -Xmx900m"
            + " org.apache.hadoop.yarn.dmlc.ApplicationMaster"
            + this.cacheFileArg + ' ' + this.appArgs + " 1>"
            + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
            + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr";

    LOG.debug(cmd);
    amContainer.setCommands(Collections.singletonList(cmd));

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(1024);
    capability.setVirtualCores(1);
    LOG.info("jobname=" + this.jobName + ",username=" + this.userName);

    appContext.setApplicationName(jobName + ":DMLC-YARN");
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue(queue);
    //appContext.setUser(userName);
    LOG.info("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED
            && appState != YarnApplicationState.KILLED
            && appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println("Application " + appId + " finished with"
            + " state " + appState + " at " + appReport.getFinishTime());
    if (!appReport.getFinalApplicationStatus().equals(
            FinalApplicationStatus.SUCCEEDED)) {
        System.err.println(appReport.getDiagnostics());
        System.out.println("Available queues:");
        for (QueueInfo q : yarnClient.getAllQueues()) {
          System.out.println(q.getQueueName());
        }

        yarnClient.killApplication(appId);
    }
}