Java JobSplit.TaskSplitMetaInfo方法代码示例

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
@Override
public Task obtainNewMapTask(final TaskTrackerStatus tts, int clusterSize,
    int numUniqueHosts, int localityLevel) throws IOException {
  for (int map = 0; map < maps.length; map++) {
    FakeTaskInProgress tip = (FakeTaskInProgress) maps[map];
    if (!tip.isRunning() && !tip.isComplete() &&
        getLocalityLevel(tip, tts) < localityLevel) {
      TaskAttemptID attemptId = getTaskAttemptID(tip);
      JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
      Task task = new MapTask("", attemptId, 0, split.getSplitIndex(), 1) {
        @Override
        public String toString() {
          return String.format("%s on %s", getTaskID(), tts.getTrackerName());
        }
      };
      runningMapTasks++;
      tip.createTaskAttempt(task, tts.getTrackerName());
      nonLocalRunningMaps.add(tip);
      taskTrackerManager.startTask(tts.getTrackerName(), task, tip);
      return task;
    }
  }
  return null;
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
public Task obtainNewMapTask(final TaskTrackerStatus tts, int clusterSize,
    int numUniqueHosts, int localityLevel) throws IOException {
  for (int map = 0; map < maps.length; map++) {
    FakeTaskInProgress tip = (FakeTaskInProgress) maps[map];
    if (!tip.isRunning() && !tip.isComplete() &&
        getLocalityLevel(tip, tts) < localityLevel) {
      TaskAttemptID attemptId = getTaskAttemptID(tip);
      JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
      Task task = new MapTask("", attemptId, 0, split.getSplitIndex(), 1) {
        @Override
        public String toString() {
          return String.format("%s on %s", getTaskID(), tts.getTrackerName());
        }
      };
      runningMapTasks++;
      tip.createTaskAttempt(task, tts.getTrackerName());
      nonLocalRunningMaps.add(tip);
      taskTrackerManager.startTask(tts.getTrackerName(), task, tip);
      return task;
    }
  }
  return null;
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
public Task obtainNewMapTask(final TaskTrackerStatus tts, int clusterSize,
    int numUniqueHosts, int localityLevel) throws IOException {
  for (int map = 0; map < maps.length; map++) {
    HFSPFakeTaskInProgress tip = (HFSPFakeTaskInProgress) maps[map];
    if (!tip.isRunning() && !tip.isComplete()
        && getLocalityLevel(tip, tts) < localityLevel) {
      TaskAttemptID attemptId = getTaskAttemptID(tip);
      JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
      Task task = new MapTask("", attemptId, 0, split.getSplitIndex(), 1) {
        @Override
        public String toString() {
          return String.format("%s on %s", getTaskID(), tts.getTrackerName());
        }
      };
      runningMapTasks++;
      tip.createTaskAttempt(task, tts.getTrackerName());
      nonLocalRunningMaps.add(tip);
      taskTrackerManager.startTask(tts.getTrackerName(), task, tip);
      return task;
    }
  }
  return null;
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
FakeTaskInProgress(
  JobID jId, JobConf jobConf, Task t,
  boolean isMap, FakeJobInProgress job, 
  JobSplit.TaskSplitMetaInfo split) {
  super(jId, "", split, null, jobConf, job, 0, 1);
  this.isMap = isMap;
  this.fakeJob = job;
  activeTasks = new TreeMap<TaskAttemptID, String>();
  activeTasks.put(t.getTaskID(), "tt");
  // create a fake status for a task that is running for a bit
  this.taskStatus = TaskStatus.createTaskStatus(isMap);
  taskStatus.setProgress(0.5f);
  taskStatus.setRunState(TaskStatus.State.RUNNING);
  if (jobConf.getMapSpeculativeExecution()) {
    //resetting of the hasSpeculativeMap is done
    //when speculative map is scheduled by the job.
    hasSpeculativeMap = true;
  }
  if (jobConf.getReduceSpeculativeExecution()) {
    //resetting of the hasSpeculativeReduce is done
    //when speculative reduce is scheduled by the job.
    hasSpeculativeReduce = true;
  }
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
public void testResourceEstimator() throws Exception {
  final int maps = 100;
  final int reduces = 2;
  final int singleMapOutputSize = 1000;
  JobConf jc = new JobConf();
  JobID jid = new JobID("testJT", 0);
  jc.setNumMapTasks(maps);
  jc.setNumReduceTasks(reduces);
  
  JobInProgress jip = new JobInProgress(jid, jc, 
      UtilsForTests.getJobTracker());
  //unfortunately, we can't set job input size from here.
  ResourceEstimator re = new ResourceEstimator(jip);

  for(int i = 0; i < maps; ++i) {
    if (i < maps / 10) {
      // re.thresholdToUse is maps / 10
      long estOutSize = re.getEstimatedMapOutputSize();
      System.out.println(estOutSize);
      assertEquals(0, estOutSize);
    }
    
    TaskStatus ts = new MapTaskStatus();
    ts.setOutputSize(singleMapOutputSize);
    JobSplit.TaskSplitMetaInfo split =
        new JobSplit.TaskSplitMetaInfo(new String[0], 0, 0);
    TaskInProgress tip = 
      new TaskInProgress(jid, "", split, jip.jobtracker, jc, jip, 0, 1);
    re.updateWithCompletedTask(ts, tip);
  }
  assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
  assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize());
  
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
FakeTaskInProgress(JobID jId, int id, JobConf jobConf,
    FakeJobInProgress job, String[] inputLocations,
    JobSplit.TaskSplitMetaInfo split) {
  super(jId, "", split, job.jobtracker, jobConf, job, id, 1);
  this.isMap = true;
  this.fakeJob = job;
  this.inputLocations = inputLocations;
  activeTasks = new TreeMap<TaskAttemptID, String>();
  taskStatus = TaskStatus.createTaskStatus(isMap);
  taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
public void testResourceEstimator() throws Exception {
  final int maps = 100;
  final int reduces = 2;
  final int singleMapOutputSize = 1000;
  JobConf jc = new JobConf();
  JobID jid = new JobID("testJT", 0);
  jc.setNumMapTasks(maps);
  jc.setNumReduceTasks(reduces);
  
  JobInProgress jip = new JobInProgress(jid, jc, 
      UtilsForTests.getJobTracker());
  //unfortunately, we can't set job input size from here.
  ResourceEstimator re = new ResourceEstimator(jip);
  
  for(int i = 0; i < maps / 10 ; ++i) {

    long estOutSize = re.getEstimatedMapOutputSize();
    System.out.println(estOutSize);
    assertEquals(0, estOutSize);
    
    TaskStatus ts = new MapTaskStatus();
    ts.setOutputSize(singleMapOutputSize);
    JobSplit.TaskSplitMetaInfo split =
        new JobSplit.TaskSplitMetaInfo(new String[0], 0, 0);
    TaskInProgress tip = 
      new TaskInProgress(jid, "", split, jip.jobtracker, jc, jip, 0, 1);
    re.updateWithCompletedTask(ts, tip);
  }
  assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
  assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize());
  
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
FakeTaskInProgress(JobID jId, int id, JobConf jobConf,
    FakeJobInProgress job, String[] inputLocations, 
    JobSplit.TaskSplitMetaInfo split, JobTracker jt) {
  super(jId, "", split, jt, jobConf, job, id, 1);
  this.isMap = true;
  this.fakeJob = job;
  this.inputLocations = inputLocations;
  activeTasks = new TreeMap<TaskAttemptID, String>();
  taskStatus = TaskStatus.createTaskStatus(isMap);
  taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
HFSPFakeTaskInProgress(JobID jId, JobTracker jobTracker, boolean isMap,
    int id, JobConf jobConf, HFSPFakeJobInProgress job,
    String[] inputLocations, JobSplit.TaskSplitMetaInfo split, FakeClock clock) {
  super(jId, "", split, jobTracker, jobConf, job, id, 1);
  this.clock = clock;
  this.isMap = isMap;
  this.fakeJob = job;
  this.inputLocations = inputLocations;
  activeTasks = new TreeMap<TaskAttemptID, String>();
  taskStatus = TaskStatus.createTaskStatus(isMap);
  taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
/**
 * Initialize tasks, including setup. 
 */
@Override
public synchronized void initTasks() throws IOException {
  super.initTasks();
  JobSplit.TaskSplitMetaInfo emptySplit = new JobSplit.TaskSplitMetaInfo();
  setup = new TaskInProgress[2];
  setup[0] = new TaskInProgress(getJobID(), "test",  emptySplit,
      jobtracker, getJobConf(), this, numMapTasks + 1, 1);
  setup[1] = new TaskInProgress(getJobID(), "test", numMapTasks,
      numReduceTasks + 1, jobtracker, getJobConf(), this, 1);
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
public void testResourceEstimator() throws Exception {
  final int maps = 100;
  final int reduces = 2;
  final int singleMapOutputSize = 1000;
  JobConf jc = new JobConf();
  JobID jid = new JobID("testJT", 0);
  jc.setNumMapTasks(maps);
  jc.setNumReduceTasks(reduces);
  
  JobInProgress jip = new JobInProgress(jid, jc, 
    UtilsForTests.getJobTracker());
  //unfortunately, we can't set job input size from here.
  ResourceEstimator re = new ResourceEstimator(jip);
  
  for(int i = 0; i < maps / 10 ; ++i) {

    long estOutSize = re.getEstimatedMapOutputSize();
    System.out.println(estOutSize);
    assertEquals(0, estOutSize);
    
    TaskStatus ts = new MapTaskStatus();
    ts.setOutputSize(singleMapOutputSize);
    JobSplit.TaskSplitMetaInfo split = 
      new JobSplit.TaskSplitMetaInfo(new String[0], 0, 0);
    TaskInProgress tip = new TaskInProgress(jid, "", split, null, jc, jip, 0, 1);
    re.updateWithCompletedTask(ts, tip);
  }
  assertEquals(2* singleMapOutputSize, re.getEstimatedMapOutputSize());
  assertEquals(2* singleMapOutputSize * maps / reduces, re.getEstimatedReduceInputSize());
  
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
FakeTaskInProgress(JobID jId, int id, JobConf jobConf,
    FakeJobInProgress job, String[] inputLocations, 
    JobSplit.TaskSplitMetaInfo split) {
  super(jId, "", split, null, jobConf, job, id, 1);
  this.isMap = true;
  this.fakeJob = job;
  this.inputLocations = inputLocations;
  activeTasks = new TreeMap<TaskAttemptID, String>();
  taskStatus = TaskStatus.createTaskStatus(isMap);
  taskStatus.setRunState(TaskStatus.State.UNASSIGNED);
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
@Override
 public synchronized void initTasks() throws IOException {
   // initTasks is needed to create non-empty cleanup and setup TIP
   // arrays, otherwise calls such as job.getTaskInProgress will fail
   JobID jobId = getJobID();
   JobConf conf = getJobConf();
   String jobFile = "";
   // create two cleanup tips, one map and one reduce.
   cleanup = new TaskInProgress[2];
   // cleanup map tip.
   cleanup[0] = new TaskInProgress(jobId, jobFile, null, 
           jobtracker, conf, this, numMapTasks, 1);
   cleanup[0].setJobCleanupTask();
   // cleanup reduce tip.
   cleanup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,
                      numReduceTasks, jobtracker, conf, this, 1);
   cleanup[1].setJobCleanupTask();
   // create two setup tips, one map and one reduce.
   setup = new TaskInProgress[2];
   // setup map tip.
   setup[0] = new TaskInProgress(jobId, jobFile, null, 
           jobtracker, conf, this, numMapTasks + 1, 1);
   setup[0].setJobSetupTask();
   // setup reduce tip.
   setup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,
                      numReduceTasks + 1, jobtracker, conf, this, 1);
   setup[1].setJobSetupTask();
   // create maps
   numMapTasks = conf.getNumMapTasks();
   maps = new TaskInProgress[numMapTasks];
   JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
   for (int i = 0; i < numMapTasks; i++) {
     String[] inputLocations = null;
     if (mapInputLocations != null)
       inputLocations = mapInputLocations[i];
     maps[i] = new FakeTaskInProgress(getJobID(), i,
         getJobConf(), this, inputLocations, split);
     if (mapInputLocations == null) // Job has no locality info
       nonLocalMaps.add(maps[i]);
   }
   // create reduces
   numReduceTasks = conf.getNumReduceTasks();
   reduces = new TaskInProgress[numReduceTasks];
   for (int i = 0; i < numReduceTasks; i++) {
     reduces[i] = new FakeTaskInProgress(getJobID(), i,
         getJobConf(), this);
   }
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
SubmittedJob(JobID jobID, String jobSubmitDirectory, Credentials credentials, Configuration configuration) throws IOException, InterruptedException {
    this.jobID = jobID;
    this.configuration = configuration;
    this.jobSubmitDirectoryPath = new Path(jobSubmitDirectory);
    this.fileSystem = FileSystem.get(configuration);

    JobSplit.TaskSplitMetaInfo splitInfo[] = SplitMetaInfoReader.readSplitMetaInfo(jobID, fileSystem, configuration, jobSubmitDirectoryPath);

    Path jobSplitFile = JobSubmissionFiles.getJobSplitFile(jobSubmitDirectoryPath);
    FSDataInputStream stream = fileSystem.open(jobSplitFile);

    for (JobSplit.TaskSplitMetaInfo info : splitInfo) {
        Object split = getSplitDetails(stream, info.getStartOffset(), configuration);
        inputSplits.add(split);
        splitLocations.put(split, info.getLocations());
        LOG.info("Adding split for execution. Split = " + split + " Locations: " + Arrays.toString(splitLocations.get(split)));
    }

    stream.close();

    jobConfPath = JobSubmissionFiles.getJobConfPath(jobSubmitDirectoryPath);

    if (!fileSystem.exists(jobConfPath)) {
        throw new IOException("Cannot find job.xml. Path = " + jobConfPath);
    }

    //We cannot just use JobConf(Path) constructor,
    //because it does not work for HDFS locations.
    //The comment in Configuration#loadResource() states,
    //for the case when the Path to the resource is provided:
    //"Can't use FileSystem API or we get an infinite loop
    //since FileSystem uses Configuration API.  Use java.io.File instead."
    //
    //Workaround: construct empty Configuration, provide it with
    //input stream and give it to JobConf constructor.
    FSDataInputStream jobConfStream = fileSystem.open(jobConfPath);
    Configuration jobXML = new Configuration(false);
    jobXML.addResource(jobConfStream);

    //The configuration does not actually gets read before we attempt to
    //read some property. Call to #size() will make Configuration to
    //read the input stream.
    jobXML.size();

    //We are done with input stream, can close it now.
    jobConfStream.close();

    jobConf = new JobConf(jobXML);

    newApi = jobConf.getUseNewMapper();


    jobStatus = new JobStatus(jobID, 0f, 0f, 0f, 0f,
            JobStatus.State.RUNNING,
            JobPriority.NORMAL,
            UserGroupInformation.getCurrentUser().getUserName(),
            jobID.toString(),
            jobConfPath.toString(), "");
}
 

import org.apache.hadoop.mapreduce.split.JobSplit; //导入方法依赖的package包/类
@Override
public synchronized void initTasks() throws IOException {
  // initTasks is needed to create non-empty cleanup and setup TIP
  // arrays, otherwise calls such as job.getTaskInProgress will fail
  JobID jobId = getJobID();
  JobConf conf = getJobConf();
  String jobFile = "";
  // create two cleanup tips, one map and one reduce.
  cleanup = new TaskInProgress[2];
  // cleanup map tip.
  cleanup[0] = new TaskInProgress(jobId, jobFile, null, 
          jobtracker, conf, this, numMapTasks, 1);
  cleanup[0].setJobCleanupTask();
  // cleanup reduce tip.
  cleanup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,
                     numReduceTasks, jobtracker, conf, this, 1);
  cleanup[1].setJobCleanupTask();
  // create two setup tips, one map and one reduce.
  setup = new TaskInProgress[2];
  // setup map tip.
  setup[0] = new TaskInProgress(jobId, jobFile, null, 
          jobtracker, conf, this, numMapTasks + 1, 1);
  setup[0].setJobSetupTask();
  // setup reduce tip.
  setup[1] = new TaskInProgress(jobId, jobFile, numMapTasks,
                     numReduceTasks + 1, jobtracker, conf, this, 1);
  setup[1].setJobSetupTask();
  // create maps
  numMapTasks = conf.getNumMapTasks();
  maps = new TaskInProgress[numMapTasks];
  // empty format
  JobSplit.TaskSplitMetaInfo split = JobSplit.EMPTY_TASK_SPLIT;
  for (int i = 0; i < numMapTasks; i++) {
    String[] inputLocations = null;
    if (mapInputLocations != null)
      inputLocations = mapInputLocations[i];
    maps[i] = new FakeTaskInProgress(getJobID(), i,
        getJobConf(), this, inputLocations, split, jobtracker);
    if (mapInputLocations == null) // Job has no locality info
      nonLocalMaps.add(maps[i]);
  }
  // create reduces
  numReduceTasks = conf.getNumReduceTasks();
  reduces = new TaskInProgress[numReduceTasks];
  for (int i = 0; i < numReduceTasks; i++) {
    reduces[i] = new FakeTaskInProgress(getJobID(), i,
        getJobConf(), this, jobtracker);
  }
  
  initialized = true;
}