Java QueueMetrics类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
    ReservationAgent agent, Resource totalCapacity, long step,
    ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
    String queueName, Planner replanner, boolean getMoveOnExpiry, Clock clock) {
  this.queueMetrics = queueMetrics;
  this.policy = policy;
  this.agent = agent;
  this.step = step;
  this.totalCapacity = totalCapacity;
  this.resCalc = resCalc;
  this.minAlloc = minAlloc;
  this.maxAlloc = maxAlloc;
  this.rleSparseVector = new RLESparseResourceAllocation(resCalc, minAlloc);
  this.queueName = queueName;
  this.replanner = replanner;
  this.getMoveOnExpiry = getMoveOnExpiry;
  this.clock = clock;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
public synchronized 
static FSQueueMetrics forQueue(String queueName, Queue parent,
    boolean enableUserMetrics, Configuration conf) {
  MetricsSystem ms = DefaultMetricsSystem.instance();
  QueueMetrics metrics = queueMetrics.get(queueName);
  if (metrics == null) {
    metrics = new FSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
        .tag(QUEUE_INFO, queueName);
    
    // Register with the MetricsSystems
    if (ms != null) {
      metrics = ms.register(
              sourceName(queueName).toString(), 
              "Metrics for queue: " + queueName, metrics);
    }
    queueMetrics.put(queueName, metrics);
  }

  return (FSQueueMetrics)metrics;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
public AbstractCSQueue(CapacitySchedulerContext cs, 
    String queueName, CSQueue parent, CSQueue old) throws IOException {
  this.labelManager = cs.getRMContext().getNodeLabelManager();
  this.parent = parent;
  this.queueName = queueName;
  this.resourceCalculator = cs.getResourceCalculator();
  
  // must be called after parent and queueName is set
  this.metrics = old != null ? old.getMetrics() :
      QueueMetrics.forQueue(getQueuePath(), parent,
          cs.getConfiguration().getEnableUserMetrics(),
          cs.getConf());

  this.csContext = cs;
  this.minimumAllocation = csContext.getMinimumResourceCapability();
  
  // initialize ResourceUsage
  queueUsage = new ResourceUsage();
  queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath());
  
  // initialize QueueCapacities
  queueCapacities = new QueueCapacities(parent == null);    
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
@Before
public void setup() throws Exception {

  // 1 sec step
  step = 1000L;

  initTime = System.currentTimeMillis();
  minAlloc = Resource.newInstance(1024, 1, 1);
  res = new DefaultResourceCalculator();
  maxAlloc = Resource.newInstance(1024 * 8, 8, 8);

  mAgent = mock(ReservationAgent.class);
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  String reservationQ = testUtil.getFullReservationQueueName();
  QueueMetrics rootQueueMetrics = mock(QueueMetrics.class);
  Resource clusterResource = testUtil.calculateClusterResource(totCont);
  ReservationSchedulerConfiguration conf = mock
      (ReservationSchedulerConfiguration.class);
  NoOverCommitPolicy policy = new NoOverCommitPolicy();
  policy.init(reservationQ, conf);

  plan =
      new InMemoryPlan(rootQueueMetrics, policy, mAgent,
          clusterResource, step, res, minAlloc, maxAlloc,
          "dedicated", null, true);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  configuration = new Configuration();
  UserGroupInformation.setConfiguration(configuration);
  configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
      + RM2_NODE_ID);
  for (String confKey : YarnConfiguration
      .getServiceAddressConfKeys(configuration)) {
    configuration.set(HAUtil.addSuffix(confKey, RM1_NODE_ID), RM1_ADDRESS);
    configuration.set(HAUtil.addSuffix(confKey, RM2_NODE_ID), RM2_ADDRESS);
    configuration.set(HAUtil.addSuffix(confKey, RM3_NODE_ID), RM3_ADDRESS);
  }

  // Enable webapp to test web-services also
  configuration.setBoolean(MockRM.ENABLE_WEBAPP, true);
  configuration.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  ClusterMetrics.destroy();
  QueueMetrics.clearQueueMetrics();
  DefaultMetricsSystem.shutdown();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
public InMemoryPlan(QueueMetrics queueMetrics, SharingPolicy policy,
    ReservationAgent agent, Resource totalCapacity, long step,
    ResourceCalculator resCalc, Resource minAlloc, Resource maxAlloc,
    String queueName, Planner replanner, boolean getMoveOnExpiry,
    RMContext rmContext, Clock clock) {
  this.queueMetrics = queueMetrics;
  this.policy = policy;
  this.agent = agent;
  this.step = step;
  this.totalCapacity = totalCapacity;
  this.resCalc = resCalc;
  this.minAlloc = minAlloc;
  this.maxAlloc = maxAlloc;
  this.rleSparseVector = new RLESparseResourceAllocation(resCalc);
  this.queueName = queueName;
  this.replanner = replanner;
  this.getMoveOnExpiry = getMoveOnExpiry;
  this.clock = clock;
  this.rmStateStore = rmContext.getStateStore();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
public synchronized static CSQueueMetrics forQueue(String queueName,
    Queue parent, boolean enableUserMetrics, Configuration conf) {
  MetricsSystem ms = DefaultMetricsSystem.instance();
  QueueMetrics metrics = queueMetrics.get(queueName);
  if (metrics == null) {
    metrics =
        new CSQueueMetrics(ms, queueName, parent, enableUserMetrics, conf)
            .tag(QUEUE_INFO, queueName);

    // Register with the MetricsSystems
    if (ms != null) {
      metrics =
          ms.register(sourceName(queueName).toString(), "Metrics for queue: "
              + queueName, metrics);
    }
    queueMetrics.put(queueName, metrics);
  }

  return (CSQueueMetrics) metrics;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
@Override
public synchronized QueueMetrics getUserMetrics(String userName) {
  if (users == null) {
    return null;
  }
  CSQueueMetrics metrics = (CSQueueMetrics) users.get(userName);
  if (metrics == null) {
    metrics = new CSQueueMetrics(metricsSystem, queueName, null, false, conf);
    users.put(userName, metrics);
    metricsSystem.register(
        sourceName(queueName).append(",user=").append(userName).toString(),
        "Metrics for user '" + userName + "' in queue '" + queueName + "'",
        ((CSQueueMetrics) metrics.tag(QUEUE_INFO, queueName)).tag(USER_INFO,
            userName));
  }
  return metrics;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
@Before
public void setUp() throws PlanningException {
  resCalc = new DefaultResourceCalculator();
  minAlloc = Resource.newInstance(1024, 1);
  maxAlloc = Resource.newInstance(64 * 1024, 20);
  totalCapacity = Resource.newInstance(100 * 1024, 100);

  clock = mock(Clock.class);
  queueMetrics = mock(QueueMetrics.class);
  policy = mock(SharingPolicy.class);
  replanner = mock(Planner.class);

  when(clock.getTime()).thenReturn(1L);

  context = ReservationSystemTestUtil.createMockRMContext();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
@Before
public void setup() throws Exception {

  // 1 sec step
  step = 1000L;

  initTime = System.currentTimeMillis();
  minAlloc = Resource.newInstance(1024, 1);
  res = new DefaultResourceCalculator();
  maxAlloc = Resource.newInstance(1024 * 8, 8);

  mAgent = mock(ReservationAgent.class);
  ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
  String reservationQ = testUtil.getFullReservationQueueName();
  QueueMetrics rootQueueMetrics = mock(QueueMetrics.class);
  Resource clusterResource = testUtil.calculateClusterResource(totCont);
  ReservationSchedulerConfiguration conf = mock
      (ReservationSchedulerConfiguration.class);
  NoOverCommitPolicy policy = new NoOverCommitPolicy();
  policy.init(reservationQ, conf);

  plan =
      new InMemoryPlan(rootQueueMetrics, policy, mAgent,
          clusterResource, step, res, minAlloc, maxAlloc,
          "dedicated", null, true);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm2, app, clusterResource, queueResource, usedResource,
    numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources = Resources.subtract(queueResource, usedResource);
  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  asserteMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  asserteMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResource.getMemory(),
    usedResource.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
private void checkFifoQueue(SchedulerApplication schedulerApp,
    Resource usedResources, Resource availableResources) throws Exception {
  FifoScheduler scheduler = (FifoScheduler) rm2.getResourceScheduler();
  // ************ check cluster used Resources ********
  assertEquals(usedResources, scheduler.getUsedResource());

  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt =
      schedulerApp.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************ check queue metrics ****************
  QueueMetrics queueMetrics = scheduler.getRootQueueMetrics();
  asserteMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
    availableResources.getVirtualCores(), usedResources.getMemory(),
    usedResources.getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; //导入依赖的package包/类
public UserMetricsInfo(final ResourceManager rm, final RMContext rmContext,
    final String user) {
  ResourceScheduler rs = rm.getResourceScheduler();
  QueueMetrics metrics = rs.getRootQueueMetrics();
  QueueMetrics userMetrics = metrics.getUserMetrics(user);
  this.userMetricsAvailable = false;

  if (userMetrics != null) {
    this.userMetricsAvailable = true;
    
    this.appsSubmitted = userMetrics.getAppsSubmitted();
    this.appsCompleted = metrics.getAppsCompleted();
    this.appsPending = metrics.getAppsPending();
    this.appsRunning = metrics.getAppsRunning();
    this.appsFailed = metrics.getAppsFailed();
    this.appsKilled = metrics.getAppsKilled();

    this.runningContainers = userMetrics.getAllocatedContainers();
    this.pendingContainers = userMetrics.getPendingContainers();
    this.reservedContainers = userMetrics.getReservedContainers();
    
    this.reservedMB = userMetrics.getReservedMB();
    this.pendingMB = userMetrics.getPendingMB();
    this.allocatedMB = userMetrics.getAllocatedMB();
  }
}