Java JobSubmissionFiles.getJobConfPath方法代码示例

import org.apache.hadoop.mapreduce.JobSubmissionFiles; //导入方法依赖的package包/类
private void uploadJobFiles(JobID id, InputSplit[] splits,
                           Path jobSubmitDir, UserGroupInformation ugi,
                           final JobConf conf)
throws Exception {
  final Path confLocation = JobSubmissionFiles.getJobConfPath(jobSubmitDir);
  FileSystem fs = ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
    public FileSystem run() throws IOException {
      return confLocation.getFileSystem(conf);
    }
  });
  JobSplitWriter.createSplitFiles(jobSubmitDir, conf, fs, splits);
  FsPermission perm = new FsPermission((short)0700);
 
  // localize conf
  DataOutputStream confOut = FileSystem.create(fs, confLocation, perm);
  conf.writeXml(confOut);
  confOut.close();
}
 

import org.apache.hadoop.mapreduce.JobSubmissionFiles; //导入方法依赖的package包/类
@Override
public void startPSServer() throws AngelException {
  try {
    setUser();
    setLocalAddr();
    Path stagingDir = AngelApps.getStagingDir(conf, userName);

    // 2.get job id
    yarnClient = YarnClient.createYarnClient();
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    yarnClient.init(yarnConf);
    yarnClient.start();
    YarnClientApplication newApp;

    newApp = yarnClient.createApplication();
    GetNewApplicationResponse newAppResponse = newApp.getNewApplicationResponse();
    appId = newAppResponse.getApplicationId();
    JobID jobId = TypeConverter.fromYarn(appId);

    Path submitJobDir = new Path(stagingDir, appId.toString());
    jtFs = submitJobDir.getFileSystem(conf);

    conf.set("hadoop.http.filter.initializers",
        "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer");
    conf.set(AngelConf.ANGEL_JOB_DIR, submitJobDir.toString());
    conf.set(AngelConf.ANGEL_JOB_ID, jobId.toString());

    setInputDirectory();
    setOutputDirectory();

    // Credentials credentials = new Credentials();
    credentials.addAll(UserGroupInformation.getCurrentUser().getCredentials());
    TokenCache.obtainTokensForNamenodes(credentials, new Path[] {submitJobDir}, conf);
    checkParameters(conf);
    handleDeprecatedParameters(conf);

    // 4.copy resource files to hdfs
    copyAndConfigureFiles(conf, submitJobDir, (short) 10);

    // 5.write configuration to a xml file
    Path submitJobFile = JobSubmissionFiles.getJobConfPath(submitJobDir);
    TokenCache.cleanUpTokenReferral(conf);
    writeConf(conf, submitJobFile);

    // 6.create am container context
    ApplicationSubmissionContext appContext =
        createApplicationSubmissionContext(conf, submitJobDir, credentials, appId);

    conf.set(AngelConf.ANGEL_JOB_LIBJARS, "");

    // 7.Submit to ResourceManager
    appId = yarnClient.submitApplication(appContext);

    // 8.get app master client
    updateMaster(10 * 60);
    
    waitForAllPS(conf.getInt(AngelConf.ANGEL_PS_NUMBER, AngelConf.DEFAULT_ANGEL_PS_NUMBER));
    LOG.info("start pss success");
  } catch (Exception x) {
    LOG.error("submit application to yarn failed.", x);
    throw new AngelException(x);
  }
}
 

import org.apache.hadoop.mapreduce.JobSubmissionFiles; //导入方法依赖的package包/类
SubmittedJob(JobID jobID, String jobSubmitDirectory, Credentials credentials, Configuration configuration) throws IOException, InterruptedException {
    this.jobID = jobID;
    this.configuration = configuration;
    this.jobSubmitDirectoryPath = new Path(jobSubmitDirectory);
    this.fileSystem = FileSystem.get(configuration);

    JobSplit.TaskSplitMetaInfo splitInfo[] = SplitMetaInfoReader.readSplitMetaInfo(jobID, fileSystem, configuration, jobSubmitDirectoryPath);

    Path jobSplitFile = JobSubmissionFiles.getJobSplitFile(jobSubmitDirectoryPath);
    FSDataInputStream stream = fileSystem.open(jobSplitFile);

    for (JobSplit.TaskSplitMetaInfo info : splitInfo) {
        Object split = getSplitDetails(stream, info.getStartOffset(), configuration);
        inputSplits.add(split);
        splitLocations.put(split, info.getLocations());
        LOG.info("Adding split for execution. Split = " + split + " Locations: " + Arrays.toString(splitLocations.get(split)));
    }

    stream.close();

    jobConfPath = JobSubmissionFiles.getJobConfPath(jobSubmitDirectoryPath);

    if (!fileSystem.exists(jobConfPath)) {
        throw new IOException("Cannot find job.xml. Path = " + jobConfPath);
    }

    //We cannot just use JobConf(Path) constructor,
    //because it does not work for HDFS locations.
    //The comment in Configuration#loadResource() states,
    //for the case when the Path to the resource is provided:
    //"Can't use FileSystem API or we get an infinite loop
    //since FileSystem uses Configuration API.  Use java.io.File instead."
    //
    //Workaround: construct empty Configuration, provide it with
    //input stream and give it to JobConf constructor.
    FSDataInputStream jobConfStream = fileSystem.open(jobConfPath);
    Configuration jobXML = new Configuration(false);
    jobXML.addResource(jobConfStream);

    //The configuration does not actually gets read before we attempt to
    //read some property. Call to #size() will make Configuration to
    //read the input stream.
    jobXML.size();

    //We are done with input stream, can close it now.
    jobConfStream.close();

    jobConf = new JobConf(jobXML);

    newApi = jobConf.getUseNewMapper();


    jobStatus = new JobStatus(jobID, 0f, 0f, 0f, 0f,
            JobStatus.State.RUNNING,
            JobPriority.NORMAL,
            UserGroupInformation.getCurrentUser().getUserName(),
            jobID.toString(),
            jobConfPath.toString(), "");
}