Java NodeType.NODE_LOCAL属性代码示例

private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

private CSAssignment createReservedIncreasedCSAssignment(
    SchedContainerChangeRequest request) {
  CSAssignment assignment =
      new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
          application, false, false);
  Resources.addTo(assignment.getAssignmentInformation().getReserved(),
      request.getDeltaCapacity());
  assignment.getAssignmentInformation().incrReservations();
  assignment.getAssignmentInformation().addReservationDetails(
      request.getContainerId(), application.getCSLeafQueue().getQueuePath());
  assignment.setIncreasedAllocation(true);
  
  LOG.info("Reserved increase container request:" + request.toString());
  
  return assignment;
}
 

private CSAssignment createSuccessfullyIncreasedCSAssignment(
    SchedContainerChangeRequest request, boolean fromReservation) {
  CSAssignment assignment =
      new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
          application, false, fromReservation);
  Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
      request.getDeltaCapacity());
  assignment.getAssignmentInformation().incrAllocations();
  assignment.getAssignmentInformation().addAllocationDetails(
      request.getContainerId(), application.getCSLeafQueue().getQueuePath());
  assignment.setIncreasedAllocation(true);
  
  // notify application
  application
      .getCSLeafQueue()
      .getOrderingPolicy()
      .containerAllocated(application,
          application.getRMContainer(request.getContainerId()));

  LOG.info("Approved increase container request:" + request.toString()
      + " fromReservation=" + fromReservation);    
  
  return assignment;
}
 

private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  //优先级是否已经达到要求以分配reservation 的 containers
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

private synchronized CSAssignment 
assignReservedContainer(FiCaSchedulerApp application, 
    FiCaSchedulerNode node, RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, false);
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

private CSAssignment createSuccessfullyIncreasedCSAssignment(
    SchedContainerChangeRequest request, boolean fromReservation) {
  CSAssignment assignment =
      new CSAssignment(request.getDeltaCapacity(), NodeType.NODE_LOCAL, null,
          application, CSAssignment.SkippedType.NONE, fromReservation);
  Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
      request.getDeltaCapacity());
  assignment.getAssignmentInformation().incrAllocations();
  assignment.getAssignmentInformation().addAllocationDetails(
      request.getContainerId(), application.getCSLeafQueue().getQueuePath());
  assignment.setIncreasedAllocation(true);
  
  // notify application
  application
      .getCSLeafQueue()
      .getOrderingPolicy()
      .containerAllocated(application,
          application.getRMContainer(request.getContainerId()));

  LOG.info("Approved increase container request:" + request.toString()
      + " fromReservation=" + fromReservation);    
  
  return assignment;
}
 

private int getMaxAllocatableContainers(FiCaSchedulerApp application,
    Priority priority, FiCaSchedulerNode node, NodeType type) {
  int maxContainers = 0;
  
  ResourceRequest offSwitchRequest = 
    application.getResourceRequest(priority, ResourceRequest.ANY);
  if (offSwitchRequest != null) {
    maxContainers = offSwitchRequest.getNumContainers();
  }

  if (type == NodeType.OFF_SWITCH) {
    return maxContainers;
  }

  if (type == NodeType.RACK_LOCAL) {
    ResourceRequest rackLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (rackLocalRequest == null) {
      return maxContainers;
    }

    maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
  }

  if (type == NodeType.NODE_LOCAL) {
    ResourceRequest nodeLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
    if (nodeLocalRequest != null) {
      maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
    }
  }

  return maxContainers;
}
 

/**
 * Return the level at which we are allowed to schedule containers, given the
 * current size of the cluster and thresholds indicating how many nodes to
 * fail at (as a fraction of cluster size) before relaxing scheduling
 * constraints.
 */
public synchronized NodeType getAllowedLocalityLevel(Priority priority,
    int numNodes, double nodeLocalityThreshold, double rackLocalityThreshold) {
  // upper limit on threshold
  if (nodeLocalityThreshold > 1.0) { nodeLocalityThreshold = 1.0; }
  if (rackLocalityThreshold > 1.0) { rackLocalityThreshold = 1.0; }

  // If delay scheduling is not being used, can schedule anywhere
  if (nodeLocalityThreshold < 0.0 || rackLocalityThreshold < 0.0) {
    return NodeType.OFF_SWITCH;
  }

  // Default level is NODE_LOCAL
  if (!allowedLocalityLevel.containsKey(priority)) {
    allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL);
    return NodeType.NODE_LOCAL;
  }

  NodeType allowed = allowedLocalityLevel.get(priority);

  // If level is already most liberal, we're done
  if (allowed.equals(NodeType.OFF_SWITCH)) return NodeType.OFF_SWITCH;

  double threshold = allowed.equals(NodeType.NODE_LOCAL) ? nodeLocalityThreshold :
    rackLocalityThreshold;

  // Relax locality constraints once we've surpassed threshold.
  if (getSchedulingOpportunities(priority) > (numNodes * threshold)) {
    if (allowed.equals(NodeType.NODE_LOCAL)) {
      allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL);
      resetSchedulingOpportunities(priority);
    }
    else if (allowed.equals(NodeType.RACK_LOCAL)) {
      allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH);
      resetSchedulingOpportunities(priority);
    }
  }
  return allowedLocalityLevel.get(priority);
}
 

public CSAssignment(FiCaSchedulerApp application, RMContainer excessReservation) {
  this.resource = excessReservation.getContainer().getResource();
  this.type = NodeType.NODE_LOCAL;
  this.application = application;
  this.excessReservation = excessReservation;
  this.skipped = false;
}
 

public boolean canSchedule(NodeType type, String hostName) {
  if (type == NodeType.NODE_LOCAL) { 
    return hosts.contains(hostName);
  } else if (type == NodeType.RACK_LOCAL) {
    return racks.contains(Application.resolve(hostName));
  } 
  
  return true;
}
 

private void updateResourceRequests(Map<String, ResourceRequest> requests, 
    NodeType type, Task task) {
  if (type == NodeType.NODE_LOCAL) {
    for (String host : task.getHosts()) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("updateResourceRequests:" + " application=" + applicationId
          + " type=" + type + " host=" + host
          + " request=" + ((requests == null) ? "null" : requests.get(host)));
      }
      updateResourceRequest(requests.get(host));
    }
  }
  
  if (type == NodeType.NODE_LOCAL || type == NodeType.RACK_LOCAL) {
    for (String rack : task.getRacks()) {
      if(LOG.isDebugEnabled()) {
        LOG.debug("updateResourceRequests:" + " application=" + applicationId
          + " type=" + type + " rack=" + rack
          + " request=" + ((requests == null) ? "null" : requests.get(rack)));
      }
      updateResourceRequest(requests.get(rack));
    }
  }
  
  updateResourceRequest(requests.get(ResourceRequest.ANY));
  
  if(LOG.isDebugEnabled()) {
    LOG.debug("updateResourceRequests:" + " application=" + applicationId
      + " #asks=" + ask.size());
  }
}
 

private synchronized CSAssignment assignContainersToChildQueues(Resource cluster, 
    FiCaSchedulerNode node, boolean needToUnreserve) {
  CSAssignment assignment = 
      new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
  
  printChildQueues();

  // Try to assign to most 'under-served' sub-queue
  for (Iterator<CSQueue> iter=childQueues.iterator(); iter.hasNext();) {
    CSQueue childQueue = iter.next();
    if(LOG.isDebugEnabled()) {
      LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
        + " stats: " + childQueue);
    }
    assignment = childQueue.assignContainers(cluster, node, needToUnreserve);
    if(LOG.isDebugEnabled()) {
      LOG.debug("Assigned to queue: " + childQueue.getQueuePath() +
        " stats: " + childQueue + " --> " + 
        assignment.getResource() + ", " + assignment.getType());
    }

    // If we do assign, remove the queue and re-insert in-order to re-sort
    if (Resources.greaterThan(
            resourceCalculator, cluster, 
            assignment.getResource(), Resources.none())) {
      // Remove and re-insert to sort
      iter.remove();
      LOG.info("Re-sorting assigned queue: " + childQueue.getQueuePath() + 
          " stats: " + childQueue);
      childQueues.add(childQueue);
      if (LOG.isDebugEnabled()) {
        printChildQueues();
      }
      break;
    }
  }
  
  return assignment;
}
 

private boolean reservationExceedsThreshold(FSSchedulerNode node,
                                               NodeType type) {
  // Only if not node-local
  if (type != NodeType.NODE_LOCAL) {
    int existingReservations = getNumReservations(node.getRackName(),
            type == NodeType.OFF_SWITCH);
    int totalAvailNodes =
            (type == NodeType.OFF_SWITCH) ? scheduler.getNumClusterNodes() :
                    scheduler.getNumNodesInRack(node.getRackName());
    int numAllowedReservations =
            (int)Math.ceil(
                    totalAvailNodes * scheduler.getReservableNodesRatio());
    if (existingReservations >= numAllowedReservations) {
      DecimalFormat df = new DecimalFormat();
      df.setMaximumFractionDigits(2);
      if (LOG.isDebugEnabled()) {
        LOG.debug("Reservation Exceeds Allowed number of nodes:" +
                " app_id=" + getApplicationId() +
                " existingReservations=" + existingReservations +
                " totalAvailableNodes=" + totalAvailNodes +
                " reservableNodesRatio=" + df.format(
                                        scheduler.getReservableNodesRatio()) +
                " numAllowedReservations=" + numAllowedReservations);
      }
      return true;
    }
  }
  return false;
}
 

private int getMaxAllocatableContainers(FiCaSchedulerApp application,
    Priority priority, FiCaSchedulerNode node, NodeType type) {
  ResourceRequest offSwitchRequest = 
    application.getResourceRequest(priority, ResourceRequest.ANY);
  int maxContainers = offSwitchRequest.getNumContainers();

  if (type == NodeType.OFF_SWITCH) {
    return maxContainers;
  }

  if (type == NodeType.RACK_LOCAL) {
    ResourceRequest rackLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (rackLocalRequest == null) {
      return maxContainers;
    }

    maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
  }

  if (type == NodeType.NODE_LOCAL) {
    ResourceRequest nodeLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
    if (nodeLocalRequest != null) {
      maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
    }
  }

  return maxContainers;
}
 

/**
 * Return the level at which we are allowed to schedule containers.
 * Given the thresholds indicating how much time passed before relaxing
 * scheduling constraints.
 */
public synchronized NodeType getAllowedLocalityLevelByTime(Priority priority,
        long nodeLocalityDelayMs, long rackLocalityDelayMs,
        long currentTimeMs) {

  // if not being used, can schedule anywhere
  if (nodeLocalityDelayMs < 0 || rackLocalityDelayMs < 0) {
    return NodeType.OFF_SWITCH;
  }

  // default level is NODE_LOCAL
  if (! allowedLocalityLevel.containsKey(priority)) {
    allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL);
    return NodeType.NODE_LOCAL;
  }

  NodeType allowed = allowedLocalityLevel.get(priority);

  // if level is already most liberal, we're done
  if (allowed.equals(NodeType.OFF_SWITCH)) {
    return NodeType.OFF_SWITCH;
  }

  // check waiting time
  long waitTime = currentTimeMs;
  if (lastScheduledContainer.containsKey(priority)) {
    waitTime -= lastScheduledContainer.get(priority);
  } else {
    waitTime -= getStartTime();
  }

  long thresholdTime = allowed.equals(NodeType.NODE_LOCAL) ?
          nodeLocalityDelayMs : rackLocalityDelayMs;

  if (waitTime > thresholdTime) {
    if (allowed.equals(NodeType.NODE_LOCAL)) {
      allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL);
      resetSchedulingOpportunities(priority, currentTimeMs);
    } else if (allowed.equals(NodeType.RACK_LOCAL)) {
      allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH);
      resetSchedulingOpportunities(priority, currentTimeMs);
    }
  }
  return allowedLocalityLevel.get(priority);
}