Java TaskCompletionEvent类代码示例

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
private static List<TaskCompletionEvent> getAllTaskCompletionEvent(Job completedJob) {
  List<TaskCompletionEvent> completionEvents = new LinkedList<>();

  while (true) {
    try {
      TaskCompletionEvent[] bunchOfEvents;
      bunchOfEvents = completedJob.getTaskCompletionEvents(completionEvents.size());
      if (bunchOfEvents == null || bunchOfEvents.length == 0) {
        break;
      }
      completionEvents.addAll(Arrays.asList(bunchOfEvents));
    } catch (IOException e) {
      break;
    }
  }

  return completionEvents;
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
/**
 * Remove all bad paths caused by speculative execution
 * The problem happens when speculative task attempt initialized but then killed in the middle of processing.
 * Some partial file was generated at {tmp_output}/_temporary/1/_temporary/attempt_xxx_xxx/part-m-xxxx.avro,
 * without being committed to its final destination at {tmp_output}/part-m-xxxx.avro.
 *
 * @param job Completed MR job
 * @param fs File system that can handle file system
 * @return all successful paths
 */
public static List<Path> removeFailedPaths(Job job, Path tmpPath, FileSystem fs) throws IOException {
  List<TaskCompletionEvent> failedEvents = CompactionAvroJobConfigurator.getUnsuccessfulTaskCompletionEvent(job);

  List<Path> allFilePaths = DatasetHelper.getApplicableFilePaths(fs, tmpPath, Lists.newArrayList("avro"));
  List<Path> goodPaths = new ArrayList<>();
  for (Path filePath: allFilePaths) {
    if (CompactionAvroJobConfigurator.isFailedPath(filePath, failedEvents)) {
      fs.delete(filePath, false);
      log.error("{} is a bad path so it was deleted", filePath);
    } else {
      goodPaths.add(filePath);
    }
  }

  return goodPaths;
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
/**
 * List the events for the given job
 * @param jobId the job id for the job's events to list
 * @throws IOException
 */
private void listEvents(Job job, int fromEventId, int numEvents)
    throws IOException, InterruptedException {
  TaskCompletionEvent[] events = job.
    getTaskCompletionEvents(fromEventId, numEvents);
  System.out.println("Task completion events for " + job.getJobID());
  System.out.println("Number of events (from " + fromEventId + ") are: " 
    + events.length);
  for(TaskCompletionEvent event: events) {
    System.out.println(event.getStatus() + " " + 
      event.getTaskAttemptId() + " " + 
      getTaskLogURL(event.getTaskAttemptId(), event.getTaskTrackerHttp()));
  }
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
@Override
public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobID, int i, int i2) throws IOException, InterruptedException {
    if (submittedJobs.containsKey(org.apache.hadoop.mapred.JobID.downgrade(jobID))) {
        return new TaskCompletionEvent[0];
    } else {
        return backupRunner.getTaskCompletionEvents(jobID, i, i2);
    }
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
/**
 * List the events for the given job
 * @param job the job to list
 * @param fromEventId event id for the job's events to list from
 * @param numEvents number of events we want to list
 * @throws IOException
 */
private void listEvents(Job job, int fromEventId, int numEvents)
    throws IOException, InterruptedException {
  TaskCompletionEvent[] events = job.
    getTaskCompletionEvents(fromEventId, numEvents);
  System.out.println("Task completion events for " + job.getJobID());
  System.out.println("Number of events (from " + fromEventId + ") are: " 
    + events.length);
  for(TaskCompletionEvent event: events) {
    System.out.println(event.getStatus() + " " + 
      event.getTaskAttemptId() + " " + 
      getTaskLogURL(event.getTaskAttemptId(), event.getTaskTrackerHttp()));
  }
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobId,
    int fromEventId, int maxEvents)
    throws IOException, InterruptedException {
  // FIXME seems like there is support in client to query task failure
  // related information
  // However, api does not make sense for DAG
  return new TaskCompletionEvent[0];
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
@Override
public TaskCompletionEvent[] getTaskCompletionEvents(JobID arg0, int arg1, int arg2)
		throws IOException, InterruptedException {
	return clientCache.getClient(arg0).getTaskCompletionEvents(arg0, arg1, arg2);
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
@Override
public TaskCompletionEvent[] getTaskCompletionEvents(JobID arg0, int arg1,
    int arg2) throws IOException, InterruptedException {
  return clientCache.getClient(arg0).getTaskCompletionEvents(arg0, arg1, arg2);
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
public MapTaskCompletionEventsUpdate getMapCompletionEvents(JobID jobId, 
    int fromEventId, int maxLocs, TaskAttemptID id) throws IOException {
  return new MapTaskCompletionEventsUpdate(
    org.apache.hadoop.mapred.TaskCompletionEvent.EMPTY_ARRAY, false);
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
public TaskCompletionEvent[] getTaskCompletionEvents(
    org.apache.hadoop.mapreduce.JobID jobid
    , int fromEventId, int maxEvents) throws IOException {
  return TaskCompletionEvent.EMPTY_ARRAY;
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
@Override
public MapTaskCompletionEventsUpdate getMapCompletionEvents(JobID jobId, 
    int fromEventId, int maxLocs, TaskAttemptID id) throws IOException {
  return new MapTaskCompletionEventsUpdate(
    org.apache.hadoop.mapred.TaskCompletionEvent.EMPTY_ARRAY, false);
}
 

import org.apache.hadoop.mapreduce.TaskCompletionEvent; //导入依赖的package包/类
/** {@inheritDoc} */
@Override public TaskCompletionEvent[] getTaskCompletionEvents(JobID jobid, int fromEventId, int maxEvents)
    throws IOException, InterruptedException {
    return new TaskCompletionEvent[0];
}