当前位置: 首页>>代码示例>>Java>>正文

Java MRApp.setClusterInfo方法代码示例

本文整理汇总了Java中org.apache.hadoop.mapreduce.v2.app.MRApp.setClusterInfo方法的典型用法代码示例。如果您正苦于以下问题：Java MRApp.setClusterInfo方法的具体用法？Java MRApp.setClusterInfo怎么用？Java MRApp.setClusterInfo使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.mapreduce.v2.app.MRApp的用法示例。


在下文中一共展示了MRApp.setClusterInfo方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: verifyMillisCounters

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1, 0)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Counters counters = job.getAllCounters();
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
  Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
  Assert.assertEquals(mapMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
  Assert.assertEquals(reduceMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
开发者ID:naver，项目名称:hadoop，代码行数:63，代码来源:TestTaskAttempt.java

示例2: verifyMillisCounters

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Counters counters = job.getAllCounters();
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
  Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
  Assert.assertEquals(mapMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
  Assert.assertEquals(reduceMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:63，代码来源:TestTaskAttempt.java

示例3: verifySlotMillis

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
public void verifySlotMillis(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
      job.getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_MAPS)
          .getValue());
  Assert.assertEquals(
      (int) Math.ceil((float) reduceMemMb / minContainerSize), job
          .getAllCounters().findCounter(JobCounter.SLOTS_MILLIS_REDUCES)
          .getValue());
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:53，代码来源:TestTaskAttempt.java

示例4: verifyMillisCounters

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Counters counters = job.getAllCounters();
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),

      counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
  Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
  Assert.assertEquals(mapMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
  Assert.assertEquals(reduceMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
开发者ID:hopshadoop，项目名称:hops，代码行数:64，代码来源:TestTaskAttempt.java


注：本文中的org.apache.hadoop.mapreduce.v2.app.MRApp.setClusterInfo方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。