Java SLSConfiguration类代码示例

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private void startAM() throws YarnException, IOException {
  // application/container configuration
  int heartbeatInterval = conf.getInt(
          SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS,
          SLSConfiguration.AM_HEARTBEAT_INTERVAL_MS_DEFAULT);
  int containerMemoryMB = conf.getInt(SLSConfiguration.CONTAINER_MEMORY_MB,
          SLSConfiguration.CONTAINER_MEMORY_MB_DEFAULT);
  int containerVCores = conf.getInt(SLSConfiguration.CONTAINER_VCORES,
          SLSConfiguration.CONTAINER_VCORES_DEFAULT);
  Resource containerResource =
          BuilderUtils.newResource(containerMemoryMB, containerVCores);

  // application workload
  if (isSLS) {
    startAMFromSLSTraces(containerResource, heartbeatInterval);
  } else {
    startAMFromRumenTraces(containerResource, heartbeatInterval);
  }
  numAMs = amMap.size();
  remainingApps = numAMs;
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
private void initMetricsCSVOutput() {
  int timeIntervalMS = conf.getInt(
          SLSConfiguration.METRICS_RECORD_INTERVAL_MS,
          SLSConfiguration.METRICS_RECORD_INTERVAL_MS_DEFAULT);
  File dir = new File(metricsOutputDir + "/metrics");
  if(! dir.exists()
          && ! dir.mkdirs()) {
    LOG.error("Cannot create directory " + dir.getAbsoluteFile());
  }
  final CsvReporter reporter = CsvReporter.forRegistry(metrics)
          .formatFor(Locale.US)
          .convertRatesTo(TimeUnit.SECONDS)
          .convertDurationsTo(TimeUnit.MILLISECONDS)
          .build(new File(metricsOutputDir + "/metrics"));
  reporter.start(timeIntervalMS, TimeUnit.MILLISECONDS);
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
private void startRM() throws IOException, ClassNotFoundException {
  Configuration rmConf = new YarnConfiguration();
  String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);

  // For CapacityScheduler we use a sub-classing instead of wrapping
  // to allow scheduler-specific invocations from monitors to work
  // this can be used for other schedulers as well if we care to
  // exercise/track behaviors that are not common to the scheduler api
  if(Class.forName(schedulerClass) == CapacityScheduler.class) {
    rmConf.set(YarnConfiguration.RM_SCHEDULER,
        SLSCapacityScheduler.class.getName());
  } else {
    rmConf.set(YarnConfiguration.RM_SCHEDULER,
            ResourceSchedulerWrapper.class.getName());
    rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
  }

  rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
  rm = new ResourceManager();
  rm.init(rmConf);
  rm.start();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  RMStorageFactory.setConfiguration(conf);
  YarnAPIStorageFactory.setConfiguration(conf);
  DBUtility.InitializeDB();

  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
public SLSRunner(boolean isSLS, String inputTraces[], String nodeFile,
                 String outputDir, Set<String> trackedApps,
                 boolean printsimulation)
        throws IOException, ClassNotFoundException {
  this.isSLS = isSLS;
  this.inputTraces = inputTraces.clone();
  this.nodeFile = nodeFile;
  this.trackedApps = trackedApps;
  this.printSimulation = printsimulation;
  metricsOutputDir = outputDir;
  
  nmMap = new HashMap<NodeId, NMSimulator>();
  queueAppNumMap = new HashMap<String, Integer>();
  amMap = new HashMap<String, AMSimulator>();
  amClassMap = new HashMap<String, Class>();
  
  // runner configuration
  conf = new Configuration(false);
  conf.addResource("sls-runner.xml");
  // runner
  int poolSize = conf.getInt(SLSConfiguration.RUNNER_POOL_SIZE, 
                              SLSConfiguration.RUNNER_POOL_SIZE_DEFAULT); 
  SLSRunner.runner.setQueueSize(poolSize);
  // <AMType, Class> map
  for (Map.Entry e : conf) {
    String key = e.getKey().toString();
    if (key.startsWith(SLSConfiguration.AM_TYPE)) {
      String amType = key.substring(SLSConfiguration.AM_TYPE.length());
      amClassMap.put(amType, Class.forName(conf.get(key)));
    }
  }
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
private void startRM() throws IOException, ClassNotFoundException {
  Configuration rmConf = new YarnConfiguration();
  String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
  rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
  rmConf.set(YarnConfiguration.RM_SCHEDULER,
          ResourceSchedulerWrapper.class.getName());
  rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
  rm = new ResourceManager();
  rm.init(rmConf);
  rm.start();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@Before
public void setup() {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
private void startRM() throws IOException, ClassNotFoundException {
  Configuration rmConf = new YarnConfiguration();
  String schedulerClass = rmConf.get(YarnConfiguration.RM_SCHEDULER);
  rmConf.set(SLSConfiguration.RM_SCHEDULER, schedulerClass);
  rmConf.set(YarnConfiguration.RM_SCHEDULER,
          ResourceSchedulerWrapper.class.getName());
  rmConf.set(SLSConfiguration.METRICS_OUTPUT_DIR, metricsOutputDir);
  RMStorageFactory.setConfiguration(rmConf);
  YarnAPIStorageFactory.setConfiguration(rmConf);
  DBUtility.InitializeDB();
  rm = new ResourceManager();
  rm.init(rmConf);
  rm.start();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  RMStorageFactory.setConfiguration(conf);
  YarnAPIStorageFactory.setConfiguration(conf);
  DBUtility.InitializeDB();
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private void initMetrics() throws Exception {
  metrics = new MetricRegistry();
  // configuration
  metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR);
  int metricsWebAddressPort = conf.getInt(
          SLSConfiguration.METRICS_WEB_ADDRESS_PORT,
          SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT);
  // create SchedulerMetrics for current scheduler
  String schedulerMetricsType = conf.get(scheduler.getClass().getName());
  Class schedulerMetricsClass = schedulerMetricsType == null?
          defaultSchedulerMetricsMap.get(scheduler.getClass()) :
          Class.forName(schedulerMetricsType);
  schedulerMetrics = (SchedulerMetrics)ReflectionUtils
          .newInstance(schedulerMetricsClass, new Configuration());
  schedulerMetrics.init(scheduler, metrics);

  // register various metrics
  registerJvmMetrics();
  registerClusterResourceMetrics();
  registerContainerAppNumMetrics();
  registerSchedulerMetrics();

  // .csv output
  initMetricsCSVOutput();

  // start web app to provide real-time tracking
  web = new SLSWebApp(this, metricsWebAddressPort);
  web.start();

  // a thread to update histogram timer
  pool = new ScheduledThreadPoolExecutor(2);
  pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
          TimeUnit.MILLISECONDS);

  // a thread to output metrics for real-tiem tracking
  pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
          TimeUnit.MILLISECONDS);

  // application running information
  jobRuntimeLogBW = new BufferedWriter(
          new FileWriter(metricsOutputDir + "/jobruntime.csv"));
  jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
          "simulate_start_time,simulate_end_time" + EOL);
  jobRuntimeLogBW.flush();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private void initMetrics() throws Exception {
  metrics = new MetricRegistry();
  // configuration
  metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR);
  int metricsWebAddressPort = conf.getInt(
          SLSConfiguration.METRICS_WEB_ADDRESS_PORT,
          SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT);
  // create SchedulerMetrics for current scheduler
  String schedulerMetricsType = conf.get(scheduler.getClass().getName());
  Class schedulerMetricsClass = schedulerMetricsType == null?
          defaultSchedulerMetricsMap.get(scheduler.getClass()) :
          Class.forName(schedulerMetricsType);
  schedulerMetrics = (SchedulerMetrics)ReflectionUtils
          .newInstance(schedulerMetricsClass, new Configuration());
  schedulerMetrics.init(scheduler, metrics);

  // register various metrics
  registerJvmMetrics();
  registerClusterResourceMetrics();
  registerContainerAppNumMetrics();
  registerSchedulerMetrics();

  // .csv output
  initMetricsCSVOutput();

  // start web app to provide real-time tracking
  web = new SLSWebApp(this, metricsWebAddressPort);
  web.start();

  // a thread to update histogram timer
  pool = new ScheduledThreadPoolExecutor(2);
  pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
          TimeUnit.MILLISECONDS);

  // a thread to output metrics for real-tiem tracking
  pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
          TimeUnit.MILLISECONDS);

  // application running information
  jobRuntimeLogBW =
      new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
          metricsOutputDir + "/jobruntime.csv"), "UTF-8"));
  jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
          "simulate_start_time,simulate_end_time" + EOL);
  jobRuntimeLogBW.flush();
}
 

import org.apache.hadoop.yarn.sls.conf.SLSConfiguration; //导入依赖的package包/类
@SuppressWarnings({ "unchecked", "rawtypes" })
private void initMetrics() throws Exception {
  metrics = new MetricRegistry();
  // configuration
  metricsOutputDir = conf.get(SLSConfiguration.METRICS_OUTPUT_DIR);
  int metricsWebAddressPort = conf.getInt(
          SLSConfiguration.METRICS_WEB_ADDRESS_PORT,
          SLSConfiguration.METRICS_WEB_ADDRESS_PORT_DEFAULT);
  // create SchedulerMetrics for current scheduler
  String schedulerMetricsType = conf.get(CapacityScheduler.class.getName());
  Class schedulerMetricsClass = schedulerMetricsType == null?
          defaultSchedulerMetricsMap.get(CapacityScheduler.class) :
          Class.forName(schedulerMetricsType);
  schedulerMetrics = (SchedulerMetrics)ReflectionUtils
          .newInstance(schedulerMetricsClass, new Configuration());
  schedulerMetrics.init(this, metrics);

  // register various metrics
  registerJvmMetrics();
  registerClusterResourceMetrics();
  registerContainerAppNumMetrics();
  registerSchedulerMetrics();

  // .csv output
  initMetricsCSVOutput();

  // start web app to provide real-time tracking
  web = new SLSWebApp(this, metricsWebAddressPort);
  web.start();

  // a thread to update histogram timer
  pool = new ScheduledThreadPoolExecutor(2);
  pool.scheduleAtFixedRate(new HistogramsRunnable(), 0, 1000,
          TimeUnit.MILLISECONDS);

  // a thread to output metrics for real-tiem tracking
  pool.scheduleAtFixedRate(new MetricsLogRunnable(), 0, 1000,
          TimeUnit.MILLISECONDS);

  // application running information
  jobRuntimeLogBW =
      new BufferedWriter(new OutputStreamWriter(new FileOutputStream(
          metricsOutputDir + "/jobruntime.csv"), "UTF-8"));
  jobRuntimeLogBW.write("JobID,real_start_time,real_end_time," +
          "simulate_start_time,simulate_end_time" + EOL);
  jobRuntimeLogBW.flush();
}