Java ResourceCalculatorProcessTree类代码示例

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
public void initialize() {
  Class<? extends ResourceCalculatorProcessTree> clazz =
      PSAgentContext
          .get()
          .getConf()
          .getClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE, null,
              ResourceCalculatorProcessTree.class);
  pTree =
      ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
          System.getenv().get("JVM_PID"), clazz, PSAgentContext.get().getConf());
  if (pTree != null) {
    pTree.updateProcessTree();
    initCpuCumulativeTime = pTree.getCumulativeCpuTime();
  }
  LOG.info(" Using ResourceCalculatorProcessTree : " + pTree);
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
private boolean isEnabled() {
  if (resourceCalculatorPlugin == null) {
          LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
      LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
              + this.getClass().getName() + " is disabled.");
          return false;
  }
  if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
    LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
        "needed. Not running the monitor-thread");
    return false;
  }

  return true;
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
private void initResourceCalculatorPlugins() {
  Class<? extends ResourceCalculatorProcessTree> clazz = amConf.getClass(
      TezConfiguration.TEZ_TASK_RESOURCE_CALCULATOR_PROCESS_TREE_CLASS,
      TezMxBeanResourceCalculator.class,
      ResourceCalculatorProcessTree.class);

  // this is set by YARN NM
  String pid = System.getenv().get("JVM_PID");
  // for the local debug test cases fallback to JVM hooks that are not portable
  if (pid == null || pid.length() == 0) {
    String processName = java.lang.management.ManagementFactory.getRuntimeMXBean().getName();
    pid = processName.split("@")[0];
  }
  cpuPlugin = ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pid, clazz, amConf);
  
  gcPlugin = new GcTimeUpdater(null);
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
public ProcessTreeInfo(ContainerId containerId, String pid,
    ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
    int cpuVcores, int gpuGcores) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
  this.cpuVcores = cpuVcores;
  this.gpuGcores = gpuGcores;
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
    String containerId, long limit) {
  long currentMemUsage = pTree.getVirtualMemorySize();
  // as processes begin with an age 1, we want to see if there are processes
  // more than 1 iteration old.
  long curMemUsageOfAgedProcesses = pTree.getVirtualMemorySize(1);
  return isProcessTreeOverLimit(containerId, currentMemUsage,
                                curMemUsageOfAgedProcesses, limit);
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
private String formatErrorMessage(String memTypeExceeded,
    long currentVmemUsage, long vmemLimit,
    long currentPmemUsage, long pmemLimit,
    String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree) {
  return
    String.format("Container [pid=%s,containerID=%s] is running beyond %s memory limits. ",
        pId, containerId, memTypeExceeded) +
    "Current usage: " +
    formatUsageString(currentVmemUsage, vmemLimit,
                      currentPmemUsage, pmemLimit) +
    ". Killing container.\n" +
    "Dump of the process-tree for " + containerId + " :\n" +
    pTree.getProcessTreeDump();
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
/**
 * Update resource information counters
 */
void updateResourceCounters() {
  // Update generic resource counters
  updateHeapUsageCounter();

  // Updating resources specified in ResourceCalculatorProcessTree
  if (pTree == null) {
    return;
  }
  pTree.updateProcessTree();
  long cpuTime = pTree.getCumulativeCpuTime();
  long pMem = pTree.getRssMemorySize();
  long vMem = pTree.getVirtualMemorySize();
  // Remove the CPU time consumed previously by JVM reuse
  if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE &&
      initCpuCumulativeTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
    cpuTime -= initCpuCumulativeTime;
  }
  
  if (cpuTime != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.CPU_MILLISECONDS).setValue(cpuTime);
  }
  
  if (pMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES).setValue(pMem);
  }

  if (vMem != ResourceCalculatorProcessTree.UNAVAILABLE) {
    counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES).setValue(vMem);
  }
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
public ProcessTreeInfo(ContainerId containerId, String pid,
    ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
    int cpuVcores) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
  this.cpuVcores = cpuVcores;
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
    String containerId, long limit) {
  long currentMemUsage = pTree.getCumulativeVmem();
  // as processes begin with an age 1, we want to see if there are processes
  // more than 1 iteration old.
  long curMemUsageOfAgedProcesses = pTree.getCumulativeVmem(1);
  return isProcessTreeOverLimit(containerId, currentMemUsage,
                                curMemUsageOfAgedProcesses, limit);
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
public ProcessTreeInfo(ContainerId containerId, String pid,
    ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
@Override
public void onJobInitialize(InvocationParameters parameters) throws IOException {
    //Take this chance to stub out ResourceCalculatorProcessTree
    parameters.getHadoopInvocationParameters().getConfiguration().setClass(MRConfig.RESOURCE_CALCULATOR_PROCESS_TREE, DummyResourceCalculatorProcessTree.class, ResourceCalculatorProcessTree.class);

    //Initialize the distributed cache
    DistributedCacheManager cacheManager = new DistributedCacheManager();
    cacheManager.setup(parameters.getHadoopInvocationParameters().getConfiguration());
    distributedCaches.put(parameters.getHadoopInvocationParameters().getJobID(), cacheManager);

    super.onJobInitialize(parameters);
}
 

import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; //导入依赖的package包/类
public ProcessTreeInfo(ContainerId containerId, String pid,
    ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit,
    int cpuVcores, int gpus) {
  this.containerId = containerId;
  this.pid = pid;
  this.pTree = pTree;
  this.vmemLimit = vmemLimit;
  this.pmemLimit = pmemLimit;
  this.cpuVcores = cpuVcores;
  this.gpus = gpus;
}