Java RMContainer.getContainer方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
public synchronized ContainersAndNMTokensAllocation
    pullNewlyAllocatedContainersAndNMTokens() {
  List<Container> returnContainerList =
      new ArrayList<Container>(newlyAllocatedContainers.size());
  List<NMToken> nmTokens = new ArrayList<NMToken>();
  for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
    .hasNext();) {
    RMContainer rmContainer = i.next();
    Container container = rmContainer.getContainer();
    try {
      // create container token and NMToken altogether.
      container.setContainerToken(rmContext.getContainerTokenSecretManager()
        .createContainerToken(container.getId(), container.getNodeId(),
          getUser(), container.getResource(), container.getPriority(),
          rmContainer.getCreationTime(), this.logAggregationContext));
      NMToken nmToken =
          rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
            getApplicationAttemptId(), container);
      if (nmToken != null) {
        nmTokens.add(nmToken);
      }
    } catch (IllegalArgumentException e) {
      // DNS might be down, skip returning this container.
      LOG.error("Error trying to assign container token and NM token to" +
          " an allocated container " + container.getId(), e);
      continue;
    }
    returnContainerList.add(container);
    i.remove();
    rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
      RMContainerEventType.ACQUIRED));
  }
  return new ContainersAndNMTokensAllocation(returnContainerList, nmTokens);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
/**
 * Clean up a completed container.
 */
@Override
protected synchronized void completedContainer(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  if (rmContainer == null) {
    LOG.info("Null container completed...");
    return;
  }

  Container container = rmContainer.getContainer();

  // Get the application for the finished container
  FSAppAttempt application =
      getCurrentAttemptForContainer(container.getId());
  ApplicationId appId =
      container.getId().getApplicationAttemptId().getApplicationId();
  if (application == null) {
    LOG.info("Container " + container + " of" +
        " unknown application attempt " + appId +
        " completed with event " + event);
    return;
  }

  // Get the node on which the container was allocated
  FSSchedulerNode node = getFSSchedulerNode(container.getNodeId());

  if (rmContainer.getState() == RMContainerState.RESERVED) {
    application.unreserve(rmContainer.getReservedPriority(), node);
  } else {
    application.containerCompleted(rmContainer, containerStatus, event);
    node.releaseContainer(container);
    updateRootQueueMetrics();
  }

  LOG.info("Application attempt " + application.getApplicationAttemptId()
      + " released container " + container.getId() + " on node: " + node
      + " with event: " + event);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
synchronized public void containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  
  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);
  
  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);
  
  // Remove from the list of containers
  liveContainers.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  this.attemptResourceUsage.decUsed(containerResource);

  // remove from preemption map if it is completed
  preemptionMap.remove(rmContainer);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
/**
 * The Scheduler has allocated containers on this node to the given
 * application.
 * 
 * @param rmContainer
 *          allocated container
 */
public synchronized void allocateContainer(RMContainer rmContainer) {
  Container container = rmContainer.getContainer();
  deductAvailableResource(container.getResource());
  ++numContainers;

  launchedContainers.put(container.getId(), rmContainer);

  LOG.info("Assigned container " + container.getId() + " of capacity "
      + container.getResource() + " on host " + rmNode.getNodeAddress()
      + ", which has " + numContainers + " containers, "
      + getUsedResource() + " used and " + getAvailableResource()
      + " available after allocation");
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
public synchronized boolean unreserve(FiCaSchedulerNode node, Priority priority) {
  Map<NodeId, RMContainer> reservedContainers =
    this.reservedContainers.get(priority);

  if (reservedContainers != null) {
    RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());

    // unreserve is now triggered in new scenarios (preemption)
    // as a consequence reservedcontainer might be null, adding NP-checks
    if (reservedContainer != null
        && reservedContainer.getContainer() != null
        && reservedContainer.getContainer().getResource() != null) {

      if (reservedContainers.isEmpty()) {
        this.reservedContainers.remove(priority);
      }
      // Reset the re-reservation count
      resetReReservations(priority);

      Resource resource = reservedContainer.getContainer().getResource();
      this.attemptResourceUsage.decReserved(node.getPartition(), resource);

      LOG.info("Application " + getApplicationId() + " unreserved "
          + " on node " + node + ", currently has "
          + reservedContainers.size() + " at priority " + priority
          + "; currentReservation " + this.attemptResourceUsage.getReserved()
          + " on node-label=" + node.getPartition());
      return true;
    }
  }
  return false;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
@Lock(CapacityScheduler.class)
@Override
protected synchronized void completedContainer(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  if (rmContainer == null) {
    LOG.info("Null container completed...");
    return;
  }
  
  Container container = rmContainer.getContainer();
  
  // Get the application for the finished container
  FiCaSchedulerApp application =
      getCurrentAttemptForContainer(container.getId());
  ApplicationId appId =
      container.getId().getApplicationAttemptId().getApplicationId();
  if (application == null) {
    LOG.info("Container " + container + " of" + " unknown application "
        + appId + " completed with event " + event);
    return;
  }
  
  // Get the node on which the container was allocated
  FiCaSchedulerNode node = getNode(container.getNodeId());
  
  // Inform the queue
  LeafQueue queue = (LeafQueue)application.getQueue();
  queue.completedContainer(clusterResource, application, node, 
      rmContainer, containerStatus, event, null, true);

  LOG.info("Application attempt " + application.getApplicationAttemptId()
      + " released container " + container.getId() + " on node: " + node
      + " with event: " + event);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
@Lock(FifoScheduler.class)
@Override
protected synchronized void completedContainer(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  if (rmContainer == null) {
    LOG.info("Null container completed...");
    return;
  }

  // Get the application for the finished container
  Container container = rmContainer.getContainer();
  FiCaSchedulerApp application =
      getCurrentAttemptForContainer(container.getId());
  ApplicationId appId =
      container.getId().getApplicationAttemptId().getApplicationId();
  
  // Get the node on which the container was allocated
  FiCaSchedulerNode node = getNode(container.getNodeId());
  
  if (application == null) {
    LOG.info("Unknown application: " + appId + 
        " released container " + container.getId() +
        " on node: " + node + 
        " with event: " + event);
    return;
  }

  // Inform the application
  application.containerCompleted(rmContainer, containerStatus, event,
      RMNodeLabelsManager.NO_LABEL);

  // Inform the node
  node.releaseContainer(container);
  
  // Update total usage
  Resources.subtractFrom(usedResource, container.getResource());

  LOG.info("Application attempt " + application.getApplicationAttemptId() + 
      " released container " + container.getId() +
      " on node: " + node + 
      " with event: " + event);
   
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
synchronized public boolean containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event,
    String partition) {

  // Remove from the list of containers
  if (null == liveContainers.remove(rmContainer.getContainerId())) {
    return false;
  }
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);

  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();

  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);

  containersToPreempt.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(),
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  attemptResourceUsage.decUsed(partition, containerResource);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;

  return true;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
private synchronized void allocateContainersToNode(FiCaSchedulerNode node) {
  if (rmContext.isWorkPreservingRecoveryEnabled()
      && !rmContext.isSchedulerReadyForAllocatingContainers()) {
    return;
  }

  // Assign new containers...
  // 1. Check for reserved applications
  // 2. Schedule if there are no reservations

  RMContainer reservedContainer = node.getReservedContainer();
  if (reservedContainer != null) {
    FiCaSchedulerApp reservedApplication =
        getCurrentAttemptForContainer(reservedContainer.getContainerId());
    
    // Try to fulfill the reservation
    LOG.info("Trying to fulfill reservation for application " + 
        reservedApplication.getApplicationId() + " on node: " + 
        node.getNodeID());
    
    LeafQueue queue = ((LeafQueue)reservedApplication.getQueue());
    CSAssignment assignment =
        queue.assignContainers(
            clusterResource,
            node,
            // TODO, now we only consider limits for parent for non-labeled
            // resources, should consider labeled resources as well.
            new ResourceLimits(labelManager.getResourceByLabel(
                RMNodeLabelsManager.NO_LABEL, clusterResource)));
    
    RMContainer excessReservation = assignment.getExcessReservation();
    if (excessReservation != null) {
    Container container = excessReservation.getContainer();
    queue.completedContainer(
        clusterResource, assignment.getApplication(), node, 
        excessReservation, 
        SchedulerUtils.createAbnormalContainerStatus(
            container.getId(), 
            SchedulerUtils.UNRESERVED_CONTAINER), 
        RMContainerEventType.RELEASED, null, true);
    }

  }

  // Try to schedule more if there are no reservations to fulfill
  if (node.getReservedContainer() == null) {
    if (calculator.computeAvailableContainers(node.getAvailableResource(),
      minimumAllocation) > 0) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Trying to schedule on node: " + node.getNodeName() +
            ", available: " + node.getAvailableResource());
      }
      root.assignContainers(
          clusterResource,
          node,
          // TODO, now we only consider limits for parent for non-labeled
          // resources, should consider labeled resources as well.
          new ResourceLimits(labelManager.getResourceByLabel(
              RMNodeLabelsManager.NO_LABEL, clusterResource)));
    }
  } else {
    LOG.info("Skipping scheduling since node " + node.getNodeID() + 
        " is reserved by application " + 
        node.getReservedContainer().getContainerId().getApplicationAttemptId()
        );
  }

}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
private Container getContainer(RMContainer rmContainer, 
    FiCaSchedulerApp application, FiCaSchedulerNode node, 
    Resource capability, Priority priority) {
  return (rmContainer != null) ? rmContainer.getContainer() :
    createContainer(application, node, capability, priority);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; //导入方法依赖的package包/类
@Override
public void completedContainer(Resource clusterResource, 
    FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, 
    ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
    boolean sortQueues) {
  if (application != null) {

    boolean removed = false;

    // Careful! Locking order is important!
    synchronized (this) {

      Container container = rmContainer.getContainer();

      // Inform the application & the node
      // Note: It's safe to assume that all state changes to RMContainer
      // happen under scheduler's lock... 
      // So, this is, in effect, a transaction across application & node
      if (rmContainer.getState() == RMContainerState.RESERVED) {
        removed = unreserve(application, rmContainer.getReservedPriority(),
            node, rmContainer);
      } else {
        removed =
            application.containerCompleted(rmContainer, containerStatus,
                event, node.getPartition());
        node.releaseContainer(container);
      }

      // Book-keeping
      if (removed) {
        releaseResource(clusterResource, application,
            container.getResource(), node.getLabels());
        LOG.info("completedContainer" +
            " container=" + container +
            " queue=" + this +
            " cluster=" + clusterResource);
      }
    }

    if (removed) {
      // Inform the parent queue _outside_ of the leaf-queue lock
      getParent().completedContainer(clusterResource, application, node,
        rmContainer, null, event, this, sortQueues);
    }
  }
}