Java TaskType.MAP属性代码示例

private void loadAllTasks() {
  if (tasksLoaded.get()) {
    return;
  }
  tasksLock.lock();
  try {
    if (tasksLoaded.get()) {
      return;
    }
    for (Map.Entry<TaskID, TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
      TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
      TaskInfo taskInfo = entry.getValue();
      Task task = new CompletedTask(yarnTaskID, taskInfo);
      tasks.put(yarnTaskID, task);
      if (task.getType() == TaskType.MAP) {
        mapTasks.put(task.getID(), task);
      } else if (task.getType() == TaskType.REDUCE) {
        reduceTasks.put(task.getID(), task);
      }
    }
    tasksLoaded.set(true);
  } finally {
    tasksLock.unlock();
  }
}
 

private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
 

private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
  // rerun previously successful map tasks
  List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
  if(taskAttemptIdList != null) {
    String mesg = "TaskAttempt killed because it ran on unusable node "
        + nodeId;
    for(TaskAttemptId id : taskAttemptIdList) {
      if(TaskType.MAP == id.getTaskId().getTaskType()) {
        // reschedule only map tasks because their outputs maybe unusable
        LOG.info(mesg + ". AttemptId:" + id);
        eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
      }
    }
  }
  // currently running task attempts on unusable nodes are handled in
  // RMContainerAllocator
}
 

private void taskKilled(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.killedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.killedReduceTaskCount++;
  }
  job.metrics.killedTask(task);
}
 

void incr(Task task) {
  TaskType type = task.getType();
  boolean finished = task.isFinished();
  if (type == TaskType.MAP) {
    if (finished) {
      ++completedMaps;
    }
    ++maps;
  } else if (type == TaskType.REDUCE) {
    if (finished) {
      ++completedReduces;
    }
    ++reduces;
  }
}
 

@Override
public long thresholdRuntime(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  TaskType type = taskID.getTaskType();

  DataStatistics statistics
      = dataStatisticsForTask(taskID);

  int completedTasksOfType
      = type == TaskType.MAP
          ? job.getCompletedMaps() : job.getCompletedReduces();

  int totalTasksOfType
      = type == TaskType.MAP
          ? job.getTotalMaps() : job.getTotalReduces();

  if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
      || (((float)completedTasksOfType) / totalTasksOfType)
            < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) {
    return Long.MAX_VALUE;
  }

  long result =  statistics == null
      ? Long.MAX_VALUE
      : (long)statistics.outlier(slowTaskRelativeTresholds.get(job));
  return result;
}
 

@SuppressWarnings("unchecked")
@Override
public void handle(ContainerAllocatorEvent event) {
  if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
    LOG.info("Processing the event " + event.toString());
    // Assign the same container ID as the AM
    ContainerId cID =
        ContainerId.newContainerId(getContext().getApplicationAttemptId(),
          this.containerId.getContainerId());
    Container container = recordFactory.newRecordInstance(Container.class);
    container.setId(cID);
    NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort);
    container.setNodeId(nodeId);
    container.setContainerToken(null);
    container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
    // send the container-assigned event to task attempt

    if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {
      JobCounterUpdateEvent jce =
          new JobCounterUpdateEvent(event.getAttemptID().getTaskId()
              .getJobId());
      // TODO Setting OTHER_LOCAL_MAP for now.
      jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
      eventHandler.handle(jce);
    }
    eventHandler.handle(new TaskAttemptContainerAssignedEvent(
        event.getAttemptID(), container, applicationACLs));
  }
}
 

private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}
 

private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
    TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
  
  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
  } else {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
  }
  if (!taskAlreadyCompleted) {
    updateMillisCounters(jce, taskAttempt);
  }
  return jce;
}
 

public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}
 

public static TaskType
    toYarn(org.apache.hadoop.mapreduce.TaskType taskType) {
  switch (taskType) {
  case MAP:
    return TaskType.MAP;
  case REDUCE:
    return TaskType.REDUCE;
  default:
    throw new YarnRuntimeException("Unrecognized task type: " + taskType);
  }
}
 

@Override
public float getProgress() {
  if (overridingState == TaskAttemptState.NEW) {
    return 0.0F;
  }
  return myAttemptID.getTaskId().getTaskType() == TaskType.MAP ? getMapProgress() : getReduceProgress();
}
 

private void taskSucceeded(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.succeededMapTaskCount++;
  } else {
    job.succeededReduceTaskCount++;
  }
  job.metrics.completedTask(task);
}
 

private int maybeScheduleASpeculation(TaskType type) {
  int successes = 0;

  long now = clock.getTime();

  ConcurrentMap<JobId, AtomicInteger> containerNeeds
      = type == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;

  for (ConcurrentMap.Entry<JobId, AtomicInteger> jobEntry : containerNeeds.entrySet()) {
    // This race conditon is okay.  If we skip a speculation attempt we
    //  should have tried because the event that lowers the number of
    //  containers needed to zero hasn't come through, it will next time.
    // Also, if we miss the fact that the number of containers needed was
    //  zero but increased due to a failure it's not too bad to launch one
    //  container prematurely.
    if (jobEntry.getValue().get() > 0) {
      continue;
    }

    int numberSpeculationsAlready = 0;
    int numberRunningTasks = 0;

    // loop through the tasks of the kind
    Job job = context.getJob(jobEntry.getKey());

    Map<TaskId, Task> tasks = job.getTasks(type);

    int numberAllowedSpeculativeTasks
        = (int) Math.max(minimumAllowedSpeculativeTasks,
            proportionTotalTasksSpeculatable * tasks.size());

    TaskId bestTaskID = null;
    long bestSpeculationValue = -1L;

    // this loop is potentially pricey.
    // TODO track the tasks that are potentially worth looking at
    for (Map.Entry<TaskId, Task> taskEntry : tasks.entrySet()) {
      long mySpeculationValue = speculationValue(taskEntry.getKey(), now);

      if (mySpeculationValue == ALREADY_SPECULATING) {
        ++numberSpeculationsAlready;
      }

      if (mySpeculationValue != NOT_RUNNING) {
        ++numberRunningTasks;
      }

      if (mySpeculationValue > bestSpeculationValue) {
        bestTaskID = taskEntry.getKey();
        bestSpeculationValue = mySpeculationValue;
      }
    }
    numberAllowedSpeculativeTasks
        = (int) Math.max(numberAllowedSpeculativeTasks,
            proportionRunningTasksSpeculatable * numberRunningTasks);

    // If we found a speculation target, fire it off
    if (bestTaskID != null
        && numberAllowedSpeculativeTasks > numberSpeculationsAlready) {
      addSpeculativeAttempt(bestTaskID);
      ++successes;
    }
  }

  return successes;
}
 

public TaskAttemptInfo(TaskAttempt ta, Boolean isRunning) {
  this(ta, TaskType.MAP, isRunning);
}