Java Resource.getVirtualCores方法代碼示例

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
private void setupLimits(ContainerId containerId,
                         Resource containerResource) throws IOException {
  String containerName = containerId.toString();

  if (isCpuWeightEnabled()) {
    int containerVCores = containerResource.getVirtualCores();
    createCgroup(CONTROLLER_CPU, containerName);
    int cpuShares = CPU_DEFAULT_WEIGHT * containerVCores;
    updateCgroup(CONTROLLER_CPU, containerName, "shares",
        String.valueOf(cpuShares));
    if (strictResourceUsageMode) {
      int nodeVCores =
          conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
      if (nodeVCores != containerVCores) {
        float containerCPU =
            (containerVCores * yarnProcessors) / (float) nodeVCores;
        int[] limits = getOverallLimits(containerCPU);
        updateCgroup(CONTROLLER_CPU, containerName, CPU_PERIOD_US,
          String.valueOf(limits[0]));
        updateCgroup(CONTROLLER_CPU, containerName, CPU_QUOTA_US,
          String.valueOf(limits[1]));
      }
    }
  }
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }

  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = resource.getMemory()
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = resource.getVirtualCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long gcoreSeconds = resource.getGpuCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds, gcoreSeconds);
  }
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
protected void refreshMaximumAllocation(Resource newMaxAlloc) {
  maxAllocWriteLock.lock();
  try {
    configuredMaximumAllocation = Resources.clone(newMaxAlloc);
    int maxMemory = newMaxAlloc.getMemory();
    if (maxNodeMemory != -1) {
      maxMemory = Math.min(maxMemory, maxNodeMemory);
    }
    int maxVcores = newMaxAlloc.getVirtualCores();
    if (maxNodeVCores != -1) {
      maxVcores = Math.min(maxVcores, maxNodeVCores);
    }
    int maxGcores = newMaxAlloc.getGpuCores();
    if (maxNodeGCores != -1) {
      maxGcores = Math.min(maxGcores, maxNodeGCores);
    }
    maximumAllocation = Resources.createResource(maxMemory, maxVcores, maxGcores);
  } finally {
    maxAllocWriteLock.unlock();
  }
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
@Override
public int compareTo(Resource other) {
  int diff = this.getMemory() - other.getMemory();
  if (diff == 0) {
    diff = this.getVirtualCores() - other.getVirtualCores();
    if (diff == 0) {
      diff = this.getGpuCores() - other.getGpuCores();
    }
  }
  return diff;
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
@Override
public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
  
  if (lhs.equals(rhs)) {
    return 0;
  }

  float[] lValues = new float[] {
    (clusterResource.getMemory() != 0) ? (float) lhs.getMemory() / clusterResource.getMemory() : lhs.getMemory(),
    (clusterResource.getVirtualCores() != 0) ? (float) lhs.getVirtualCores() / clusterResource.getVirtualCores() : lhs.getVirtualCores(),
    (clusterResource.getGpuCores() != 0) ? (float) lhs.getGpuCores() / clusterResource.getGpuCores() : 0.0f };
  Arrays.sort(lValues);

  float[] rValues = new float[] {
    (clusterResource.getMemory() != 0) ? (float) rhs.getMemory() / clusterResource.getMemory() : rhs.getMemory(),
    (clusterResource.getVirtualCores() != 0) ? (float) rhs.getVirtualCores() / clusterResource.getVirtualCores() : rhs.getVirtualCores(),
    (clusterResource.getGpuCores() != 0) ? (float) rhs.getGpuCores() / clusterResource.getGpuCores() : 0.0f };
  Arrays.sort(rValues);

  int diff = 0;
  for(int i = 0; i < 3; i++) {
    float l = lValues[i];
    float r = rValues[i];
    if (l < r) {
      diff = -1;
    } else if (l > r) {
      diff = 1;
    }
  }
  
  return diff;
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
@Override
public boolean isInvalidDivisor(Resource r) {
  if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f || r.getGpuCores() == 0.0f) {
    return true;
  }
  return false;
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
public ContainerInfo(final Context nmContext, final Container container,
     String requestUri, String pathPrefix) {

  this.id = container.getContainerId().toString();
  this.nodeId = nmContext.getNodeId().toString();
  ContainerStatus containerData = container.cloneAndGetContainerStatus();
  this.exitCode = containerData.getExitStatus();
  this.exitStatus =
      (this.exitCode == ContainerExitStatus.INVALID) ?
          "N/A" : String.valueOf(exitCode);
  this.state = container.getContainerState().toString();
  this.diagnostics = containerData.getDiagnostics();
  if (this.diagnostics == null || this.diagnostics.isEmpty()) {
    this.diagnostics = "";
  }

  this.user = container.getUser();
  Resource res = container.getResource();
  if (res != null) {
    this.totalMemoryNeededMB = res.getMemory();
    this.totalVCoresNeeded = res.getVirtualCores();
  }
  this.containerLogsShortLink = ujoin("containerlogs", this.id,
      container.getUser());

  if (requestUri == null) {
    requestUri = "";
  }
  if (pathPrefix == null) {
    pathPrefix = "";
  }
  this.containerLogsLink = join(requestUri, pathPrefix,
      this.containerLogsShortLink);
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
static boolean canFit(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  
  if(mem0 <= mem1 && cpu0 <= cpu1 && gpu0 <= gpu1) {
    return true;
  }
  return false; 
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
private AggregateAppResourceUsage calculateContainerResourceMetrics(
    RMContainer rmContainer) {
  Resource resource = rmContainer.getContainer().getResource();
  long usedMillis =
      rmContainer.getFinishTime() - rmContainer.getCreationTime();
  long memorySeconds = resource.getMemory()
                        * usedMillis / DateUtils.MILLIS_PER_SECOND;
  long vcoreSeconds = resource.getVirtualCores()
                        * usedMillis / DateUtils.MILLIS_PER_SECOND;
  long gcoreSeconds = resource.getGpuCores()
                        * usedMillis / DateUtils.MILLIS_PER_SECOND;
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds, gcoreSeconds);
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
private static int getResourceValue(Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    return resource.getMemory();
  case CPU:
    return resource.getVirtualCores();
  case GPU:
    return resource.getGpuCores();
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
@Override
public synchronized void reinitialize(
    CSQueue newlyParsedQueue, Resource clusterResource) 
throws IOException {
  // Sanity check
  if (!(newlyParsedQueue instanceof LeafQueue) || 
      !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
    throw new IOException("Trying to reinitialize " + getQueuePath() + 
        " from " + newlyParsedQueue.getQueuePath());
  }

  LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;

  // don't allow the maximum allocation to be decreased in size
  // since we have already told running AM's the size
  Resource oldMax = getMaximumAllocation();
  Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
  if (newMax.getMemory() < oldMax.getMemory()
      || newMax.getVirtualCores() < oldMax.getVirtualCores()
      || newMax.getGpuCores() < oldMax.getGpuCores()) {
    throw new IOException(
        "Trying to reinitialize "
            + getQueuePath()
            + " the maximum allocation size can not be decreased!"
            + " Current setting: " + oldMax
            + ", trying to set it to: " + newMax);
  }

  setupQueueConfigs(clusterResource);

  // queue metrics are updated, more resource may be available
  // activate the pending applications if possible
  activateApplications();
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
public static boolean fitsIn(Resource smaller, Resource bigger) {
  return smaller.getMemory() <= bigger.getMemory() &&
      smaller.getVirtualCores() <= bigger.getVirtualCores() &&
      smaller.getGpuCores() <= bigger.getGpuCores();
}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
/** 
 *  Return a command to execute the given command in OS shell.
 *  On Windows, the passed in groupId can be used to launch
 *  and associate the given groupId in a process group. On
 *  non-Windows, groupId is ignored. 
 */
protected String[] getRunCommand(String command, String groupId,
    String userName, Path pidFile, Configuration conf, Resource resource) {
  boolean containerSchedPriorityIsSet = false;
  int containerSchedPriorityAdjustment = 
      YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY;

  if (conf.get(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY) != 
      null) {
    containerSchedPriorityIsSet = true;
    containerSchedPriorityAdjustment = conf 
        .getInt(YarnConfiguration.NM_CONTAINER_EXECUTOR_SCHED_PRIORITY, 
        YarnConfiguration.DEFAULT_NM_CONTAINER_EXECUTOR_SCHED_PRIORITY);
  }

  if (Shell.WINDOWS) {
    int cpuRate = -1;
    int memory = -1;
    if (resource != null) {
      if (conf
          .getBoolean(
              YarnConfiguration.NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED,
              YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_MEMORY_LIMIT_ENABLED)) {
        memory = resource.getMemory();
      }

      if (conf.getBoolean(
          YarnConfiguration.NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED,
          YarnConfiguration.DEFAULT_NM_WINDOWS_CONTAINER_CPU_LIMIT_ENABLED)) {
        int containerVCores = resource.getVirtualCores();
        int nodeVCores = conf.getInt(YarnConfiguration.NM_VCORES,
            YarnConfiguration.DEFAULT_NM_VCORES);
        // cap overall usage to the number of cores allocated to YARN
        int nodeCpuPercentage = Math
            .min(
                conf.getInt(
                    YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT,
                    YarnConfiguration.DEFAULT_NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT),
                100);
        nodeCpuPercentage = Math.max(0, nodeCpuPercentage);
        if (nodeCpuPercentage == 0) {
          String message = "Illegal value for "
              + YarnConfiguration.NM_RESOURCE_PERCENTAGE_PHYSICAL_CPU_LIMIT
              + ". Value cannot be less than or equal to 0.";
          throw new IllegalArgumentException(message);
        }
        float yarnVCores = (nodeCpuPercentage * nodeVCores) / 100.0f;
        // CPU should be set to a percentage * 100, e.g. 20% cpu rate limit
        // should be set as 20 * 100. The following setting is equal to:
        // 100 * (100 * (vcores / Total # of cores allocated to YARN))
        cpuRate = Math.min(10000,
            (int) ((containerVCores * 10000) / yarnVCores));
      }
    }
    return new String[] { Shell.WINUTILS, "task", "create", "-m",
        String.valueOf(memory), "-c", String.valueOf(cpuRate), groupId,
        "cmd /c " + command };
  } else {
    List<String> retCommand = new ArrayList<String>();
    if (containerSchedPriorityIsSet) {
      retCommand.addAll(Arrays.asList("nice", "-n",
          Integer.toString(containerSchedPriorityAdjustment)));
    }
    retCommand.addAll(Arrays.asList("bash", command));
    return retCommand.toArray(new String[retCommand.size()]);
  }

}
 

import org.apache.hadoop.yarn.api.records.Resource; //導入方法依賴的package包/類
public IntegralResource(Resource resource) {
  this.memory = resource.getMemory();
  this.vcores = resource.getVirtualCores();
  this.gcores = resource.getGpuCores();
}