Java FiCaSchedulerNode类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private int assignContainersOnNode(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority 
) {
  // Data-local
  int nodeLocalContainers = 
    assignNodeLocalContainers(node, application, priority); 

  // Rack-local
  int rackLocalContainers = 
    assignRackLocalContainers(node, application, priority);

  // Off-switch
  int offSwitchContainers =
    assignOffSwitchContainers(node, application, priority);


  LOG.debug("assignContainersOnNode:" +
      " node=" + node.getRMNode().getNodeAddress() + 
      " application=" + application.getApplicationId().getId() +
      " priority=" + priority.getPriority() + 
      " #assigned=" + 
      (nodeLocalContainers + rackLocalContainers + offSwitchContainers));


  return (nodeLocalContainers + rackLocalContainers + offSwitchContainers);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private int assignNodeLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getNodeName());
  if (request != null) {
    // Don't allocate on this node if we don't need containers on this rack
    ResourceRequest rackRequest =
        application.getResourceRequest(priority, 
            node.getRMNode().getRackName());
    if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.NODE_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.NODE_LOCAL);
  }
  return assignedContainers;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private int assignRackLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getRMNode().getRackName());
  if (request != null) {
    // Don't allocate on this rack if the application doens't need containers
    ResourceRequest offSwitchRequest =
        application.getResourceRequest(priority, ResourceRequest.ANY);
    if (offSwitchRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.RACK_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.RACK_LOCAL);
  }
  return assignedContainers;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private synchronized void removeNode(RMNode nodeInfo) {
  FiCaSchedulerNode node = getNode(nodeInfo.getNodeID());
  if (node == null) {
    return;
  }
  // Kill running containers
  for(RMContainer container : node.getRunningContainers()) {
    completedContainer(container, 
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(), 
            SchedulerUtils.LOST_CONTAINER),
            RMContainerEventType.KILL);
  }
  
  //Remove the node
  this.nodes.remove(nodeInfo.getNodeID());
  updateMaximumAllocation(node, false);
  
  // Update cluster metrics
  Resources.subtractFrom(clusterResource, node.getRMNode().getTotalCapability());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
  }
  if (parent != null) {
    parent.recoverContainer(clusterResource, attempt, rmContainer);
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.releaseResource(clusterResource,
        rmContainer.getContainer().getResource(),
        node.getLabels());
    LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.detachContainer(clusterResource, application, rmContainer);
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName, nodeManager.getNodeLabels());
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());

  // update this node to node label manager
  if (labelManager != null) {
    labelManager.activateNode(nodeManager.getNodeID(),
        nodeManager.getTotalCapability());
  }
  
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
  int numNodes = numNodeManagers.incrementAndGet();
  updateMaximumAllocation(schedulerNode, true);
  
  LOG.info("Added node " + nodeManager.getNodeAddress() + 
      " clusterResource: " + clusterResource);

  if (scheduleAsynchronously && numNodes == 1) {
    asyncSchedulerThread.beginSchedule();
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private synchronized CSAssignment assignReservedContainer(
    FiCaSchedulerApp application, FiCaSchedulerNode node,
    RMContainer rmContainer, Resource clusterResource) {
  // Do we still need this reservation?
  Priority priority = rmContainer.getReservedPriority();
  if (application.getTotalRequiredResources(priority) == 0) {
    // Release
    return new CSAssignment(application, rmContainer);
  }

  // Try to assign if we have sufficient resources
  assignContainersOnNode(clusterResource, node, application, priority, 
      rmContainer, new ResourceLimits(Resources.none()));
  
  // Doesn't matter... since it's already charged for at time of reservation
  // "re-reservation" is *free*
  return new CSAssignment(Resources.none(), NodeType.NODE_LOCAL);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    allocateResource(clusterResource, application, rmContainer.getContainer()
        .getResource(), node.getLabels());
    LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
        + " resource=" + rmContainer.getContainer().getResource()
        + " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
        + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
        + queueUsage.getUsed() + " cluster=" + clusterResource);
    // Inform the parent queue
    getParent().attachContainer(clusterResource, application, rmContainer);
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    releaseResource(clusterResource, application, rmContainer.getContainer()
        .getResource(), node.getLabels());
    LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
        + " resource=" + rmContainer.getContainer().getResource()
        + " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
        + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
        + queueUsage.getUsed() + " cluster=" + clusterResource);
    // Inform the parent queue
    getParent().detachContainer(clusterResource, application, rmContainer);
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
public static FiCaSchedulerNode getMockNode(
    String host, String rack, int port, int capability) {
  NodeId nodeId = mock(NodeId.class);
  when(nodeId.getHost()).thenReturn(host);
  when(nodeId.getPort()).thenReturn(port);
  RMNode rmNode = mock(RMNode.class);
  when(rmNode.getNodeID()).thenReturn(nodeId);
  when(rmNode.getTotalCapability()).thenReturn(
      Resources.createResource(capability, 1));
  when(rmNode.getNodeAddress()).thenReturn(host+":"+port);
  when(rmNode.getHostName()).thenReturn(host);
  when(rmNode.getRackName()).thenReturn(rack);
  
  FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
  LOG.info("node = " + host + " avail=" + node.getAvailableResource());
  return node;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private boolean checkHeadroom(Resource clusterResource,
    ResourceLimits currentResourceLimits, Resource required,
    FiCaSchedulerNode node) {
  // If headroom + currentReservation < required, we cannot allocate this
  // require
  Resource resourceCouldBeUnReserved = application.getCurrentReservation();
  if (!application.getCSLeafQueue().getReservationContinueLooking()
      || !node.getPartition().equals(RMNodeLabelsManager.NO_LABEL)) {
    // If we don't allow reservation continuous looking, OR we're looking at
    // non-default node partition, we won't allow to unreserve before
    // allocation.
    resourceCouldBeUnReserved = Resources.none();
  }
  return Resources.greaterThanOrEqual(rc, clusterResource, Resources.add(
      currentResourceLimits.getHeadroom(), resourceCouldBeUnReserved),
      required);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
private ContainerAllocation allocate(Resource clusterResource,
    FiCaSchedulerNode node, SchedulingMode schedulingMode,
    ResourceLimits resourceLimits, Priority priority,
    RMContainer reservedContainer) {
  ContainerAllocation result =
      preAllocation(clusterResource, node, schedulingMode, resourceLimits,
          priority, reservedContainer);

  if (AllocationState.ALLOCATED == result.state
      || AllocationState.RESERVED == result.state) {
    result =
        doAllocation(result, clusterResource, node, schedulingMode, priority,
            reservedContainer);
  }

  return result;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; //导入依赖的package包/类
@Override
public void completedContainer(Resource clusterResource,
    FiCaSchedulerApp application, FiCaSchedulerNode node, 
    RMContainer rmContainer, ContainerStatus containerStatus, 
    RMContainerEventType event, CSQueue completedChildQueue,
    boolean sortQueues) {
  if (application != null) {
    internalReleaseResource(clusterResource, node,
        rmContainer.getContainer().getResource(), false, completedChildQueue,
        sortQueues);

    // Inform the parent
    if (parent != null) {
      // complete my parent
      parent.completedContainer(clusterResource, application, 
          node, rmContainer, null, event, this, sortQueues);
    }    
  }
}