Java SystemMetricsPublisher类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
    throws IOException {
  Dispatcher dispatcher = mock(Dispatcher.class);
  when(rmContext.getDispatcher()).thenReturn(dispatcher);
  EventHandler eventHandler = mock(EventHandler.class);
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queInfo.setQueueName("testqueue");
  when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
      .thenReturn(queInfo);
  when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
      .thenThrow(new IOException("queue does not exist"));
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
      getSchedulerApps(apps));
   ResourceScheduler rs = mock(ResourceScheduler.class);
   when(rmContext.getScheduler()).thenReturn(rs);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
private void mockRMContext(YarnScheduler yarnScheduler, RMContext rmContext)
    throws IOException {
  Dispatcher dispatcher = mock(Dispatcher.class);
  when(rmContext.getDispatcher()).thenReturn(dispatcher);
  EventHandler eventHandler = mock(EventHandler.class);
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  QueueInfo queInfo = recordFactory.newRecordInstance(QueueInfo.class);
  queInfo.setQueueName("testqueue");
  when(yarnScheduler.getQueueInfo(eq("testqueue"), anyBoolean(), anyBoolean()))
      .thenReturn(queInfo);
  when(yarnScheduler.getQueueInfo(eq("nonexistentqueue"), anyBoolean(), anyBoolean()))
      .thenThrow(new IOException("queue does not exist"));
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(yarnScheduler.getAppsInQueue(eq("testqueue"))).thenReturn(
      getSchedulerApps(apps));
   ResourceScheduler rs = mock(ResourceScheduler.class);
   when(rmContext.getScheduler()).thenReturn(rs);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  context = new MockRMContext();
  context.setDispatcher(TestObjectFactory.getMockDispatcher());
  context.setSystemMetricsPublisher(new SystemMetricsPublisher());
  
  nodeOne = TestObjectFactory.getRMNode("localhost-one", 8800, Resource.newInstance(1024, 2));
  nodeTwo = TestObjectFactory.getRMNode("localhost-two", 8800, Resource.newInstance(2048, 4));
  sNodeOne = new FSSchedulerNode(nodeOne, false);
  sNodeTwo = new FSSchedulerNode(nodeTwo, false);

  store = new NodeStore();
  store.add(sNodeOne);
  store.add(sNodeTwo);

  containerOne = TestObjectFactory.getRMContainer(nodeOne, context, 1, 2, 1024);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
public RMContext mockRMContext(int n, long time) {
  final List<RMApp> apps = newRMApps(n, time, RMAppState.FINISHED);
  final ConcurrentMap<ApplicationId, RMApp> map = Maps.newConcurrentMap();
  for (RMApp app : apps) {
    map.put(app.getApplicationId(), app);
  }
  Dispatcher rmDispatcher = new AsyncDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = new ContainerAllocationExpirer(
      rmDispatcher);
  AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(
      rmDispatcher);
  AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(
      rmDispatcher);
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext context = new RMContextImpl(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      null, null, null, null, null, writer) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return map;
    }
  };
  ((RMContextImpl)context).setStateStore(mock(RMStateStore.class));
  metricsPublisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)context).setSystemMetricsPublisher(metricsPublisher);
  return context;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
public RMContext mockRMContext(int n, long time) {
  final List<RMApp> apps = newRMApps(n, time, RMAppState.FINISHED);
  final ConcurrentMap<ApplicationId, RMApp> map = Maps.newConcurrentMap();
  for (RMApp app : apps) {
    map.put(app.getApplicationId(), app);
  }
  Dispatcher rmDispatcher = new AsyncDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = new ContainerAllocationExpirer(
      rmDispatcher);
  AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(
      rmDispatcher);
  AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(
      rmDispatcher);
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext context = new RMContextImpl(rmDispatcher,
      containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
      null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return map;
    }
  };
  ((RMContextImpl)context).setStateStore(mock(RMStateStore.class));
  metricsPublisher = mock(SystemMetricsPublisher.class);
  context.setSystemMetricsPublisher(metricsPublisher);
  context.setRMApplicationHistoryWriter(writer);
  return context;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  InlineDispatcher rmDispatcher = new InlineDispatcher();

  rmContext =
      new RMContextImpl(rmDispatcher, null, null, null,
        null, null, null, null, null);
  rmContext.setSystemMetricsPublisher(new SystemMetricsPublisher());
  rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));

  scheduler = mock(YarnScheduler.class);
  doAnswer(
      new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
          final SchedulerEvent event = (SchedulerEvent)(invocation.getArguments()[0]);
          eventType = event.getType();
          if (eventType == SchedulerEventType.NODE_UPDATE) {
            //DO NOTHING
          }
          return null;
        }
      }
      ).when(scheduler).handle(any(SchedulerEvent.class));

  rmDispatcher.register(SchedulerEventType.class,
      new TestSchedulerEventDispatcher());

  appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Test (timeout = 180000)
public void testStoreAllContainerMetrics() throws Exception {
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  conf.setBoolean(
      YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
      true);
  MockRM rm1 = new MockRM(conf);

  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  rm1.getRMContext().setSystemMetricsPublisher(publisher);

  rm1.start();
  MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
  RMApp app1 = rm1.submitApp(1024);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING);

  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 = ContainerId.newContainerId(
      am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
  am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
      .getAllocatedContainers();
  rm1.waitForState(nm1, containerId2, RMContainerState.ACQUIRED);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.COMPLETE);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(nm1, containerId2, RMContainerState.COMPLETED);
  rm1.stop();

  // RMContainer should be publishing system metrics for all containers.
  // Since there is 1 AM container and 1 non-AM container, there should be 2
  // container created events and 2 container finished events.
  verify(publisher, times(2)).containerCreated(any(RMContainer.class), anyLong());
  verify(publisher, times(2)).containerFinished(any(RMContainer.class), anyLong());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Test (timeout = 180000)
public void testStoreOnlyAMContainerMetrics() throws Exception {
  Configuration conf = new Configuration();
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  conf.setBoolean(
      YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
      false);
  MockRM rm1 = new MockRM(conf);

  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  rm1.getRMContext().setSystemMetricsPublisher(publisher);

  rm1.start();
  MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
  RMApp app1 = rm1.submitApp(1024);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING);

  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 = ContainerId.newContainerId(
      am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
  am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
      .getAllocatedContainers();
  rm1.waitForState(nm1, containerId2, RMContainerState.ACQUIRED);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.COMPLETE);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(nm1, containerId2, RMContainerState.COMPLETED);
  rm1.stop();

  // RMContainer should be publishing system metrics only for AM container.
  verify(publisher, times(1)).containerCreated(any(RMContainer.class), anyLong());
  verify(publisher, times(1)).containerFinished(any(RMContainer.class), anyLong());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  ((RMContextImpl) rmContext).setScheduler(scheduler);
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  FifoScheduler scheduler = new FifoScheduler();
  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Override
@Before
public void setUp() throws Exception {
  super.setUp();
  this.baseStateStoreDirectory = "/tmp/yarn-node-capacity-manager-test";
  context = new MockRMContext();
  context.setDispatcher(TestObjectFactory.getMockDispatcher());
  context.setSystemMetricsPublisher(new SystemMetricsPublisher());
  
  nodeOne = TestObjectFactory.getRMNode("localhost-one", 8800, Resource.newInstance(2048, 4));
  nodeTwo = TestObjectFactory.getRMNode("localhost-two", 8800, Resource.newInstance(1024, 2));
  sNodeOne = new FSSchedulerNode(nodeOne, false);
  sNodeTwo = new FSSchedulerNode(nodeTwo, false);

  containerOne = TestObjectFactory.getRMContainer(nodeOne, context, 1, 2, 1024);
  store = new NodeStore();
  store.add(sNodeOne);
  store.add(sNodeTwo);
  
  MyriadDriver driver = TestObjectFactory.getMyriadDriver(new MockSchedulerDriver());
  olManager = new OfferLifecycleManager(store, driver);
  state = TestObjectFactory.getSchedulerState(new MyriadConfiguration(), "/tmp/yarn-node-capacity-manager-test");
  MyriadFairScheduler scheduler = TestObjectFactory.getMyriadFairScheduler(context);
  
  scheduler.addNode(sNodeOne);
  scheduler.addNode(sNodeTwo);
  manager = new YarnNodeCapacityManager(new CompositeInterceptor(), scheduler, 
            context, driver, olManager, store, state, new TaskUtils(this.cfg));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Private
@Unstable
public void setSystemMetricsPublisher(
    SystemMetricsPublisher systemMetricsPublisher) {
  this.systemMetricsPublisher = systemMetricsPublisher;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
@Private
@Unstable
public SystemMetricsPublisher getSystemMetricsPublisher() {
  return systemMetricsPublisher;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher; //导入依赖的package包/类
protected SystemMetricsPublisher createSystemMetricsPublisher() {
  return new SystemMetricsPublisher(); 
}