Java JobStory.getJobConf方法代码示例

import org.apache.hadoop.tools.rumen.JobStory; //导入方法依赖的package包/类
/**
 * For the job to be simulated, identify the needed distributed cache files by
 * mapping original cluster's distributed cache file paths to the simulated cluster's
 * paths and add these paths in the map {@code distCacheFiles}.
 *<br>
 * JobStory should contain distributed cache related properties like
 * <li> {@link MRJobConfig#CACHE_FILES}
 * <li> {@link MRJobConfig#CACHE_FILE_VISIBILITIES}
 * <li> {@link MRJobConfig#CACHE_FILES_SIZES}
 * <li> {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}
 * <li> {@link MRJobConfig#CLASSPATH_FILES}
 *
 * <li> {@link MRJobConfig#CACHE_ARCHIVES}
 * <li> {@link MRJobConfig#CACHE_ARCHIVES_VISIBILITIES}
 * <li> {@link MRJobConfig#CACHE_ARCHIVES_SIZES}
 * <li> {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}
 * <li> {@link MRJobConfig#CLASSPATH_ARCHIVES}
 *
 * <li> {@link MRJobConfig#CACHE_SYMLINK}
 *
 * @param jobdesc JobStory of original job obtained from trace
 * @throws IOException
 */
void updateHDFSDistCacheFilesList(JobStory jobdesc) throws IOException {

  // Map original job's distributed cache file paths to simulated cluster's
  // paths, to be used by this simulated job.
  JobConf jobConf = jobdesc.getJobConf();

  String[] files = jobConf.getStrings(MRJobConfig.CACHE_FILES);
  if (files != null) {

    String[] fileSizes = jobConf.getStrings(MRJobConfig.CACHE_FILES_SIZES);
    String[] visibilities =
      jobConf.getStrings(MRJobConfig.CACHE_FILE_VISIBILITIES);
    String[] timeStamps =
      jobConf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS);

    FileSystem fs = FileSystem.get(conf);
    String user = jobConf.getUser();
    for (int i = 0; i < files.length; i++) {
      // Check if visibilities are available because older hadoop versions
      // didn't have public, private Distributed Caches separately.
      boolean visibility =
          (visibilities == null) ? true : Boolean.valueOf(visibilities[i]);
      if (isLocalDistCacheFile(files[i], user, visibility)) {
        // local FS based distributed cache file.
        // Create this file on the pseudo local FS on the fly (i.e. when the
        // simulated job is submitted).
        continue;
      }
      // distributed cache file on hdfs
      String mappedPath = mapDistCacheFilePath(files[i], timeStamps[i],
                                               visibility, user);

      // No need to add a distributed cache file path to the list if
      // (1) the mapped path is already there in the list OR
      // (2) the file with the mapped path already exists.
      // In any of the above 2 cases, file paths, timestamps, file sizes and
      // visibilities match. File sizes should match if file paths and
      // timestamps match because single file path with single timestamp
      // should correspond to a single file size.
      if (distCacheFiles.containsKey(mappedPath) ||
          fs.exists(new Path(mappedPath))) {
        continue;
      }
      distCacheFiles.put(mappedPath, Long.valueOf(fileSizes[i]));
    }
  }
}
 

import org.apache.hadoop.tools.rumen.JobStory; //导入方法依赖的package包/类
@SuppressWarnings("deprecation")
  public SimulatorJobInProgress(JobID jobid, JobTracker jobtracker,
      JobConf default_conf, JobStory jobStory) {
    super(jobid, jobStory.getJobConf(), jobtracker);
    // jobSetupCleanupNeeded set to false in parent cstr, though
    // default is true

    restartCount = 0;
    jobSetupCleanupNeeded = false;
      
    this.memoryPerMap = conf.getMemoryForMapTask();
    this.memoryPerReduce = conf.getMemoryForReduceTask();
    this.maxTaskFailuresPerTracker = conf.getMaxTaskFailuresPerTracker();
    
    this.jobId = jobid;
    String url = "http://" + jobtracker.getJobTrackerMachine() + ":"
        + jobtracker.getInfoPort() + "/jobdetails.jsp?jobid=" + jobid;
    this.jobtracker = jobtracker;
    this.conf = jobStory.getJobConf();
    this.priority = conf.getJobPriority();
    Path jobDir = jobtracker.getSystemDirectoryForJob(jobid);
    this.jobFile = new Path(jobDir, "job.xml");
    this.status = new JobStatus(jobid, 0.0f, 0.0f, 0.0f, 0.0f, JobStatus.PREP,
            priority, conf.getUser());
    this.profile = new JobProfile(jobStory.getUser(), jobid, this.jobFile
        .toString(), url, jobStory.getName(), conf.getQueueName());
    this.startTime = JobTracker.getClock().getTime();
    status.setStartTime(startTime);
    this.resourceEstimator = new ResourceEstimator(this);

    this.numMapTasks = jobStory.getNumberMaps();
    this.numReduceTasks = jobStory.getNumberReduces();
    this.taskCompletionEvents = new ArrayList<TaskCompletionEvent>(numMapTasks
        + numReduceTasks + 10);

    this.mapFailuresPercent = conf.getMaxMapTaskFailuresPercent();
    this.reduceFailuresPercent = conf.getMaxReduceTaskFailuresPercent();
    MetricsContext metricsContext = MetricsUtil.getContext("mapred");
    this.jobMetrics = MetricsUtil.createRecord(metricsContext, "job");
    this.jobMetrics.setTag("user", conf.getUser());
    this.jobMetrics.setTag("sessionId", conf.getSessionId());
    this.jobMetrics.setTag("jobName", conf.getJobName());
    this.jobMetrics.setTag("jobId", jobid.toString());

    this.maxLevel = jobtracker.getNumTaskCacheLevels();
    this.anyCacheLevel = this.maxLevel + 1;
    this.nonLocalMaps = new LinkedList<TaskInProgress>();
    this.nonLocalRunningMaps = new LinkedHashSet<TaskInProgress>();
    this.runningMapCache = new IdentityHashMap<Node, Set<TaskInProgress>>();
    this.nonRunningReduces = new LinkedList<TaskInProgress>();
    this.runningReduces = new LinkedHashSet<TaskInProgress>();
    this.slowTaskThreshold = Math.max(0.0f, conf.getFloat(
        "mapred.speculative.execution.slowTaskThreshold", 1.0f));
    this.speculativeCap = conf.getFloat(
        "mapred.speculative.execution.speculativeCap", 0.1f);
    this.slowNodeThreshold = conf.getFloat(
        "mapred.speculative.execution.slowNodeThreshold", 1.0f);

    this.jobStory = jobStory;
//    this.jobHistory = this.jobtracker.getJobHistory();
  }
 

import org.apache.hadoop.tools.rumen.JobStory; //导入方法依赖的package包/类
/**
 * For the job to be simulated, identify the needed distributed cache files by
 * mapping original cluster's distributed cache file paths to the simulated cluster's
 * paths and add these paths in the map {@code distCacheFiles}.
 *<br>
 * JobStory should contain distributed cache related properties like
 * <li> {@link MRJobConfig#CACHE_FILES}
 * <li> {@link MRJobConfig#CACHE_FILE_VISIBILITIES}
 * <li> {@link MRJobConfig#CACHE_FILES_SIZES}
 * <li> {@link MRJobConfig#CACHE_FILE_TIMESTAMPS}
 * <li> {@link MRJobConfig#CLASSPATH_FILES}
 *
 * <li> {@link MRJobConfig#CACHE_ARCHIVES}
 * <li> {@link MRJobConfig#CACHE_ARCHIVES_VISIBILITIES}
 * <li> {@link MRJobConfig#CACHE_ARCHIVES_SIZES}
 * <li> {@link MRJobConfig#CACHE_ARCHIVES_TIMESTAMPS}
 * <li> {@link MRJobConfig#CLASSPATH_ARCHIVES}
 *
 * <li> {@link MRJobConfig#CACHE_SYMLINK}
 *
 * @param jobdesc JobStory of original job obtained from trace
 * @throws IOException
 */
void updateHDFSDistCacheFilesList(JobStory jobdesc) throws IOException {

  // Map original job's distributed cache file paths to simulated cluster's
  // paths, to be used by this simulated job.
  JobConf jobConf = jobdesc.getJobConf();

  String[] files = jobConf.getStrings(MRJobConfig.CACHE_FILES);
  if (files != null) {

    String[] fileSizes = jobConf.getStrings(MRJobConfig.CACHE_FILES_SIZES);
    String[] visibilities =
      jobConf.getStrings(MRJobConfig.CACHE_FILE_VISIBILITIES);
    String[] timeStamps =
      jobConf.getStrings(MRJobConfig.CACHE_FILE_TIMESTAMPS);

    FileSystem fs = FileSystem.get(conf);
    String user = jobConf.getUser();
    for (int i = 0; i < files.length; i++) {
      // Check if visibilities are available because older hadoop versions
      // didn't have public, private Distributed Caches separately.
      boolean visibility =
          (visibilities == null) || Boolean.parseBoolean(visibilities[i]);
      if (isLocalDistCacheFile(files[i], user, visibility)) {
        // local FS based distributed cache file.
        // Create this file on the pseudo local FS on the fly (i.e. when the
        // simulated job is submitted).
        continue;
      }
      // distributed cache file on hdfs
      String mappedPath = mapDistCacheFilePath(files[i], timeStamps[i],
                                               visibility, user);

      // No need to add a distributed cache file path to the list if
      // (1) the mapped path is already there in the list OR
      // (2) the file with the mapped path already exists.
      // In any of the above 2 cases, file paths, timestamps, file sizes and
      // visibilities match. File sizes should match if file paths and
      // timestamps match because single file path with single timestamp
      // should correspond to a single file size.
      if (distCacheFiles.containsKey(mappedPath) ||
          fs.exists(new Path(mappedPath))) {
        continue;
      }
      distCacheFiles.put(mappedPath, Long.valueOf(fileSizes[i]));
    }
  }
}