Java ResourceUsageMetrics类代码示例

import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; //导入依赖的package包/类
ResourceUsageMatcherRunner(final TaskInputOutputContext context, 
                           ResourceUsageMetrics metrics) {
  Configuration conf = context.getConfiguration();
  
  // set the resource calculator plugin
  Class<? extends ResourceCalculatorPlugin> clazz =
    conf.getClass(TTConfig.TT_RESOURCE_CALCULATOR_PLUGIN,
                  null, ResourceCalculatorPlugin.class);
  ResourceCalculatorPlugin plugin = 
    ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
  
  // set the other parameters
  this.sleepTime = conf.getLong(SLEEP_CONFIG, DEFAULT_SLEEP_TIME);
  progress = new BoostingProgress(context);
  
  // instantiate a resource-usage-matcher
  matcher = new ResourceUsageMatcher();
  matcher.configure(conf, plugin, metrics, progress);
}
 

import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; //导入依赖的package包/类
@Override
public void initialize(Configuration conf, ResourceUsageMetrics metrics,
                       ResourceCalculatorPlugin monitor,
                       Progressive progress) {
  this.monitor = monitor;
  this.progress = progress;
  
  // get the target CPU usage
  targetCpuUsage = metrics.getCumulativeCpuUsage();
  if (targetCpuUsage <= 0 ) {
    enabled = false;
    return;
  } else {
    enabled = true;
  }
  
  emulationInterval =  conf.getFloat(CPU_EMULATION_PROGRESS_INTERVAL, 
                                     DEFAULT_EMULATION_FREQUENCY);
  
  // calibrate the core cpu-usage utility
  emulatorCore.calibrate(monitor, targetCpuUsage);
  
  // initialize the states
  lastSeenProgress = 0;
  lastSeenCpuUsage = 0;
}
 

import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; //导入依赖的package包/类
public LoadSplit(CombineFileSplit cfsplit, int maps, int id, long inputBytes, 
                 long inputRecords, long outputBytes, long outputRecords, 
                 double[] reduceBytes, double[] reduceRecords, 
                 long[] reduceOutputBytes, long[] reduceOutputRecords,
                 ResourceUsageMetrics metrics,
                 ResourceUsageMetrics[] rMetrics)
throws IOException {
  super(cfsplit);
  this.id = id;
  this.maps = maps;
  reduces = reduceBytes.length;
  this.inputRecords = inputRecords;
  this.outputBytes = outputBytes;
  this.outputRecords = outputRecords;
  this.reduceBytes = reduceBytes;
  this.reduceRecords = reduceRecords;
  nSpec = reduceOutputBytes.length;
  this.reduceOutputBytes = reduceOutputBytes;
  this.reduceOutputRecords = reduceOutputRecords;
  this.mapMetrics = metrics;
  this.reduceMetrics = rMetrics;
}
 

import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; //导入依赖的package包/类
private LoadSplit getLoadSplit() throws Exception {

    Path[] files = {new Path("one"), new Path("two")};
    long[] start = {1, 2};
    long[] lengths = {100, 200};
    String[] locations = {"locOne", "loctwo"};

    CombineFileSplit cfSplit = new CombineFileSplit(files, start, lengths,
            locations);
    ResourceUsageMetrics metrics = new ResourceUsageMetrics();
    metrics.setCumulativeCpuUsage(200);
    ResourceUsageMetrics[] rMetrics = {metrics};

    double[] reduceBytes = {8.1d, 8.2d};
    double[] reduceRecords = {9.1d, 9.2d};
    long[] reduceOutputBytes = {101L, 102L};
    long[] reduceOutputRecords = {111L, 112L};

    return new LoadSplit(cfSplit, 2, 1, 4L, 5L, 6L, 7L,
            reduceBytes, reduceRecords, reduceOutputBytes, reduceOutputRecords,
            metrics, rMetrics);
  }
 

import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; //导入依赖的package包/类
private static void testEmulationAccuracy(Configuration conf, 
                      FakeCpuUsageEmulatorCore fakeCore,
                      ResourceCalculatorPlugin monitor,
                      ResourceUsageMetrics metrics,
                      CumulativeCpuUsageEmulatorPlugin cpuPlugin,
                      long expectedTotalCpuUsage, long expectedTotalNumCalls) 
throws Exception {
  FakeProgressive fakeProgress = new FakeProgressive();
  fakeCore.reset();
  cpuPlugin.initialize(conf, metrics, monitor, fakeProgress);
  int numLoops = 0;
  while (fakeProgress.getProgress() < 1) {
    ++numLoops;
    float progress = (float)numLoops / 100;
    fakeProgress.setProgress(progress);
    cpuPlugin.emulate();
  }
  
  // test if the resource plugin shows the expected invocations
  assertEquals("Cumulative cpu usage emulator plugin failed (num calls)!", 
               expectedTotalNumCalls, fakeCore.getNumCalls(), 0L);
  // test if the resource plugin shows the expected usage
  assertEquals("Cumulative cpu usage emulator plugin failed (total usage)!", 
               expectedTotalCpuUsage, fakeCore.getCpuUsage(), 0L);
}
 

import org.apache.hadoop.tools.rumen.ResourceUsageMetrics; //导入依赖的package包/类
private static void testEmulationAccuracy(Configuration conf, 
                      FakeHeapUsageEmulatorCore fakeCore,
                      ResourceCalculatorPlugin monitor,
                      ResourceUsageMetrics metrics,
                      TotalHeapUsageEmulatorPlugin heapPlugin,
                      long expectedTotalHeapUsageInMB,
                      long expectedTotalNumCalls)
throws Exception {
  FakeProgressive fakeProgress = new FakeProgressive();
  fakeCore.resetFake();
  heapPlugin.initialize(conf, metrics, monitor, fakeProgress);
  int numLoops = 0;
  while (fakeProgress.getProgress() < 1) {
    ++numLoops;
    float progress = numLoops / 100.0F;
    fakeProgress.setProgress(progress);
    heapPlugin.emulate();
  }
  
  // test if the resource plugin shows the expected usage
  assertEquals("Cumulative heap usage emulator plugin failed (total usage)!", 
               expectedTotalHeapUsageInMB, fakeCore.getHeapUsageInMB(), 1L);
  // test if the resource plugin shows the expected num calls
  assertEquals("Cumulative heap usage emulator plugin failed (num calls)!", 
               expectedTotalNumCalls, fakeCore.getNumCalls(), 0L);
}