Java TaskTracker.unreserveSlots方法代码示例

import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; //导入方法依赖的package包/类
@Override
public List<Task> assignTasks(TaskTracker tt) {
  if (unreserveSlots) {
    tt.unreserveSlots(TaskType.MAP, fakeJob);
    tt.unreserveSlots(TaskType.REDUCE, fakeJob);
  } else {
    int currCount = 1;
    if (reservedCounts.containsKey(tt)) {
      currCount = reservedCounts.get(tt) + 1;
    }
    reservedCounts.put(tt, currCount);
    tt.reserveSlots(TaskType.MAP, fakeJob, currCount);
    tt.reserveSlots(TaskType.REDUCE, fakeJob, currCount);
  }
  return new ArrayList<Task>();  
}
 

import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; //导入方法依赖的package包/类
/**
 * Note that a task has failed on a given tracker and add the tracker  
 * to the blacklist iff too many trackers in the cluster i.e. 
 * (clusterSize * CLUSTER_BLACKLIST_PERCENT) haven't turned 'flaky' already.
 * 
 * @param taskTracker task-tracker on which a task failed
 */
synchronized void addTrackerTaskFailure(String trackerName, 
                                        TaskTracker taskTracker) {
  if (flakyTaskTrackers < (clusterSize * CLUSTER_BLACKLIST_PERCENT)) { 
    String trackerHostName = convertTrackerNameToHostName(trackerName);

    Integer trackerFailures = trackerToFailuresMap.get(trackerHostName);
    if (trackerFailures == null) {
      trackerFailures = 0;
    }
    trackerToFailuresMap.put(trackerHostName, ++trackerFailures);

    // Check if this tasktracker has turned 'flaky'
    if (trackerFailures.intValue() == maxTaskFailuresPerTracker) {
      ++flakyTaskTrackers;
      
      // Cancel reservations if appropriate
      if (taskTracker != null) {
        if (trackersReservedForMaps.containsKey(taskTracker)) {
          taskTracker.unreserveSlots(TaskType.MAP, this);
        }
        if (trackersReservedForReduces.containsKey(taskTracker)) {
          taskTracker.unreserveSlots(TaskType.REDUCE, this);
        }
      }
      LOG.info("TaskTracker at '" + trackerHostName + "' turned 'flaky'");
    }
  }
}
 

import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; //导入方法依赖的package包/类
/**
 * Note that a task has failed on a given tracker and add the tracker
 * to the blacklist iff too many trackers in the cluster i.e.
 * (clusterSize * CLUSTER_BLACKLIST_PERCENT) haven't turned 'flaky' already.
 *
 * @param taskTracker task-tracker on which a task failed
 */
synchronized void addTrackerTaskFailure(String trackerName,
                                        TaskTracker taskTracker,
                                        String lastFailureReason) {
  if (flakyTaskTrackers < (clusterSize * CLUSTER_BLACKLIST_PERCENT)) {
    String trackerHostName = convertTrackerNameToHostName(trackerName);

    List<String> trackerFailures = trackerToFailuresMap.get(trackerHostName);

    if (trackerFailures == null) {
      trackerFailures = new LinkedList<String>();
      trackerToFailuresMap.put(trackerHostName, trackerFailures);
    }
    trackerFailures.add(lastFailureReason);

    // Check if this tasktracker has turned 'flaky'
    if (trackerFailures.size() == maxTaskFailuresPerTracker) {
      ++flakyTaskTrackers;

      // Cancel reservations if appropriate
      if (taskTracker != null) {
        if (trackersReservedForMaps.containsKey(taskTracker)) {
          taskTracker.unreserveSlots(TaskType.MAP, this);
        }
        if (trackersReservedForReduces.containsKey(taskTracker)) {
          taskTracker.unreserveSlots(TaskType.REDUCE, this);
        }
      }
      LOG.info("TaskTracker at '" + trackerHostName + "' turned 'flaky'");
    }
  }
}
 

import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; //导入方法依赖的package包/类
private TaskLookupResult assignTasks(TaskTracker taskTracker, 
                                     int availableSlots, 
                                     boolean assignOffSwitch) 
throws IOException {
  TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();

  printQueues();

  // Check if this tasktracker has been reserved for a job...
  JobInProgress job = taskTracker.getJobForFallowSlot(type);
  if (job != null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(job.getJobID() + ": Checking 'reserved' tasktracker " + 
                taskTracker.getTrackerName() + " with " + availableSlots + 
                " '" + type + "' slots");
    }

    if (availableSlots >= job.getNumSlotsPerTask(type)) {
      // Unreserve 
      taskTracker.unreserveSlots(type, job);
      
      // We found a suitable job. Get task from it.
      if (type == TaskType.MAP) {
        // Don't care about locality!
        job.overrideSchedulingOpportunities();
      }
      return obtainNewTask(taskTrackerStatus, job, true);
    } else {
      // Re-reserve the current tasktracker
      taskTracker.reserveSlots(type, job, availableSlots);
      
      if (LOG.isDebugEnabled()) {
        LOG.debug(job.getJobID() + ": Re-reserving " + 
                  taskTracker.getTrackerName());
      }

      return TaskLookupResult.getMemFailedResult(); 
    }
  }
  
  
  for (CapacitySchedulerQueue queue : queuesForAssigningTasks) {
    //This call is for optimization if we are already over the
    //maximum-capacity we avoid traversing the queues.
    if (!queue.assignSlotsToQueue(type, 1)) {
      continue;
    }
    
    TaskLookupResult tlr = 
      getTaskFromQueue(taskTracker, availableSlots, queue, assignOffSwitch);
    TaskLookupResult.LookUpStatus lookUpStatus = tlr.getLookUpStatus();

    if (lookUpStatus == TaskLookupResult.LookUpStatus.NO_TASK_FOUND) {
      continue; // Look in other queues.
    }

    // if we find a task, return
    if (lookUpStatus == TaskLookupResult.LookUpStatus.LOCAL_TASK_FOUND ||
        lookUpStatus == TaskLookupResult.LookUpStatus.OFF_SWITCH_TASK_FOUND) {
      return tlr;
    }
    // if there was a memory mismatch, return
    else if (lookUpStatus == 
      TaskLookupResult.LookUpStatus.TASK_FAILING_MEMORY_REQUIREMENT) {
        return tlr;
    }
  }

  // nothing to give
  return TaskLookupResult.getNoTaskFoundResult();
}
 

import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; //导入方法依赖的package包/类
private TaskLookupResult assignTasks(TaskTracker taskTracker) 
throws IOException {
  TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();

  printQSIs();

  // Check if this tasktracker has been reserved for a job...
  JobInProgress job = taskTracker.getJobForFallowSlot(type);
  if (job != null) {
    int availableSlots = taskTracker.getAvailableSlots(type);
    if (LOG.isDebugEnabled()) {
      LOG.debug(job.getJobID() + ": Checking 'reserved' tasktracker " + 
                taskTracker.getTrackerName() + " with " + availableSlots + 
                " '" + type + "' slots");
    }

    if (availableSlots >= job.getNumSlotsPerTask(type)) {
      // Unreserve 
      taskTracker.unreserveSlots(type, job);
      
      // We found a suitable job. Get task from it.
      Task t = obtainNewTask(taskTrackerStatus, job);
      //if there is a task return it immediately.
      if (t != null) {
        if (LOG.isDebugEnabled()) {
          LOG.info(job.getJobID() + ": Got " + t.getTaskID() + 
                   " for reserved tasktracker " + 
                   taskTracker.getTrackerName());
        }
        // we're successful in getting a task
        return TaskLookupResult.getTaskFoundResult(t);
      } 
    } else {
      // Re-reserve the current tasktracker
      taskTracker.reserveSlots(type, job, availableSlots);
      
      if (LOG.isDebugEnabled()) {
        LOG.debug(job.getJobID() + ": Re-reserving " + 
                  taskTracker.getTrackerName());
      }

      return TaskLookupResult.getMemFailedResult(); 
    }
  }
  
  
  for (QueueSchedulingInfo qsi : qsiForAssigningTasks) {
    // we may have queues with capacity=0. We shouldn't look at jobs from 
    // these queues
    if (0 == getTSI(qsi).getCapacity()) {
      continue;
    }
    
    //This call is for optimization if we are already over the
    //maximum-capacity we avoid traversing the queues.
    if(this.areTasksInQueueOverMaxCapacity(qsi,1)) {
      continue;
    }
    TaskLookupResult tlr = getTaskFromQueue(taskTracker, qsi);
    TaskLookupResult.LookUpStatus lookUpStatus = tlr.getLookUpStatus();

    if (lookUpStatus == TaskLookupResult.LookUpStatus.NO_TASK_FOUND) {
      continue; // Look in other queues.
    }

    // if we find a task, return
    if (lookUpStatus == TaskLookupResult.LookUpStatus.TASK_FOUND) {
      return tlr;
    }
    // if there was a memory mismatch, return
    else if (lookUpStatus == 
      TaskLookupResult.LookUpStatus.TASK_FAILING_MEMORY_REQUIREMENT) {
        return tlr;
    }
  }

  // nothing to give
  return TaskLookupResult.getNoTaskFoundResult();
}
 

import org.apache.hadoop.mapreduce.server.jobtracker.TaskTracker; //导入方法依赖的package包/类
private TaskLookupResult assignTasks(TaskTracker taskTracker)
throws IOException {
  TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();

  printQSCs();

  // Check if this tasktracker has been reserved for a job...
  JobInProgress job = taskTracker.getJobForFallowSlot(type);
  if (job != null) {
    int availableSlots = taskTracker.getAvailableSlots(type);
    if (LOG.isDebugEnabled()) {
      LOG.debug(job.getJobID() + ": Checking 'reserved' tasktracker " +
                taskTracker.getTrackerName() + " with " + availableSlots +
                " '" + type + "' slots");
    }

    if (availableSlots >= job.getNumSlotsPerTask(type)) {
      // Unreserve
      taskTracker.unreserveSlots(type, job);

      // We found a suitable job. Get task from it.
      Task t = obtainNewTask(taskTrackerStatus, job);
      //if there is a task return it immediately.
      if (t != null) {
        if (LOG.isDebugEnabled()) {
          LOG.info(job.getJobID() + ": Got " + t.getTaskID() +
                   " for reserved tasktracker " +
                   taskTracker.getTrackerName());
        }
        // we're successful in getting a task
        return TaskLookupResult.getTaskFoundResult(t);
      }
    } else {
      // Re-reserve the current tasktracker
      taskTracker.reserveSlots(type, job, availableSlots);

      if (LOG.isDebugEnabled()) {
        LOG.debug(job.getJobID() + ": Re-reserving " +
                  taskTracker.getTrackerName());
      }

      return TaskLookupResult.getMemFailedResult();
    }
  }

  for (AbstractQueue q : getOrderedJobQueues()) {
    QueueSchedulingContext qsc = q.getQueueSchedulingContext();
    // we may have queues with capacity=0. We shouldn't look at jobs from
    // these queues
    if (0 == getTSC(qsc).getCapacity()) {
      continue;
    }

    //This call is important for optimization purposes , if we
    //have reached the limit already no need for traversing the queue.
    if(this.areTasksInQueueOverMaxCapacity(qsc,1)) {
      continue;
    }
    
    TaskLookupResult tlr = getTaskFromQueue(taskTracker, qsc);
    TaskLookupResult.LookUpStatus lookUpStatus = tlr.getLookUpStatus();

    if (lookUpStatus == TaskLookupResult.LookUpStatus.NO_TASK_FOUND) {
      continue; // Look in other queues.
    }

    // if we find a task, return
    if (lookUpStatus == TaskLookupResult.LookUpStatus.TASK_FOUND) {
      return tlr;
    }
    // if there was a memory mismatch, return
    else if (lookUpStatus ==
      TaskLookupResult.LookUpStatus.TASK_FAILING_MEMORY_REQUIREMENT) {
        return tlr;
    }
  }

  // nothing to give
  return TaskLookupResult.getNoTaskFoundResult();
}