Java RMNodeStatusEvent类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
private RMNodeStatusEvent getMockRMNodeStatusEvent(
    List<ContainerStatus> containerStatus) {
  NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);

  NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
  Boolean yes = new Boolean(true);
  doReturn(yes).when(healthStatus).getIsNodeHealthy();
  
  RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
  doReturn(healthStatus).when(event).getNodeHealthStatus();
  doReturn(response).when(event).getLatestResponse();
  doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
  if (containerStatus != null) {
    doReturn(containerStatus).when(event).getContainers();
  }
  return event;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test
public void testIncrementNodeCapacityUnderCapacity() throws Exception {
  resetNodeTotalCapability(nodeOne, 0, 0);
  resetNodeTotalCapability(nodeTwo, 2, 512);
  Offer offerOne = TestObjectFactory.getOffer("localhost-one", "slave-one", "mock", "offer-one", 1.0, 512.0);
  Offer offerTwo = TestObjectFactory.getOffer("localhost-two", "slave-two", "mock", "offer-two", 3.0, 1024.0);
  olManager.addOffers(offerOne);
  olManager.addOffers(offerTwo);
      
  RMNodeStatusEvent eventOne = getRMStatusEvent(nodeOne);
  handler.beforeRMNodeEventHandled(eventOne, context);
  RMNodeStatusEvent eventTwo = getRMStatusEvent(nodeTwo);
  handler.beforeRMNodeEventHandled(eventTwo, context);
  
  assertEquals(512, nodeOne.getTotalCapability().getMemory());
  assertEquals(1, nodeOne.getTotalCapability().getVirtualCores());
  assertEquals(1024, nodeTwo.getTotalCapability().getMemory());
  assertEquals(3, nodeTwo.getTotalCapability().getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test
public void testIncrementNodeCapacityOverCapacity() throws Exception {
  resetNodeTotalCapability(nodeOne, 1, 512);
  resetNodeTotalCapability(nodeTwo, 2, 2048);
  
  //Test over memory upper limit
  Offer offerOne = TestObjectFactory.getOffer("localhost-one", "slave-one", "mock", "offer-one", 0.2, 3072.0);
  //Test over CPU cores upper limit
  Offer offerTwo = TestObjectFactory.getOffer("localhost-two", "slave-two", "mock", "offer-two", 8.0, 1024.0);
  olManager.addOffers(offerOne);
  olManager.addOffers(offerTwo);

  RMNodeStatusEvent eventOne = getRMStatusEvent(nodeOne);  
  handler.beforeRMNodeEventHandled(eventOne, context);
  RMNodeStatusEvent eventTwo = getRMStatusEvent(nodeTwo);
  handler.beforeRMNodeEventHandled(eventTwo, context);
  
  assertEquals(512, nodeOne.getTotalCapability().getMemory());
  assertEquals(1, nodeOne.getTotalCapability().getVirtualCores()); 
  assertEquals(2048, nodeTwo.getTotalCapability().getMemory());
  assertEquals(2, nodeTwo.getTotalCapability().getVirtualCores());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test
public void testNodesDefaultWithUnHealthyNode() throws JSONException,
    Exception {

  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1235", 5121);
  rm.sendNodeStarted(nm1);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);

  MockNM nm3 = rm.registerNode("h3:1236", 5122);
  rm.NMwaitForState(nm3.getNodeId(), NodeState.NEW);
  rm.sendNodeStarted(nm3);
  rm.NMwaitForState(nm3.getNodeId(), NodeState.RUNNING);
  RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
      .get(nm3.getNodeId());
  NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
      "test health report", System.currentTimeMillis());
  node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeHealth,
      new ArrayList<ContainerStatus>(), null, null));
  rm.NMwaitForState(nm3.getNodeId(), NodeState.UNHEALTHY);

  ClientResponse response =
      r.path("ws").path("v1").path("cluster").path("nodes")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  // 3 nodes, including the unhealthy node and the new node.
  assertEquals("incorrect number of elements", 3, nodeArray.length());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
private RMNodeStatusEvent getMockRMNodeStatusEvent() {
  NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);

  NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
  Boolean yes = new Boolean(true);
  doReturn(yes).when(healthStatus).getIsNodeHealthy();
  
  RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
  doReturn(healthStatus).when(event).getNodeHealthStatus();
  doReturn(response).when(event).getLatestResponse();
  doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
  return event;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test (timeout = 5000)
public void testExpiredContainer() {
  // Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  verify(scheduler).handle(any(NodeAddedSchedulerEvent.class));
  
  // Expire a container
  ContainerId completedContainerId = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  node.handle(new RMNodeCleanContainerEvent(null, completedContainerId));
  Assert.assertEquals(1, node.getContainersToCleanUp().size());
  
  // Now verify that scheduler isn't notified of an expired container
  // by checking number of 'completedContainers' it got in the previous event
  RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
  ContainerStatus containerStatus = mock(ContainerStatus.class);
  doReturn(completedContainerId).when(containerStatus).getContainerId();
  doReturn(Collections.singletonList(containerStatus)).
      when(statusEvent).getContainers();
  node.handle(statusEvent);
  /* Expect the scheduler call handle function 2 times
   * 1. RMNode status from new to Running, handle the add_node event
   * 2. handle the node update event
   */
  verify(scheduler,times(2)).handle(any(NodeUpdateSchedulerEvent.class));     
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test (timeout = 5000)
public void testStatusChange(){
  //Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  //Add info to the queue first
  node.setNextHeartBeat(false);

  ContainerId completedContainerId1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  ContainerId completedContainerId2 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 1);
      
  RMNodeStatusEvent statusEvent1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEvent2 = getMockRMNodeStatusEvent();

  ContainerStatus containerStatus1 = mock(ContainerStatus.class);
  ContainerStatus containerStatus2 = mock(ContainerStatus.class);

  doReturn(completedContainerId1).when(containerStatus1).getContainerId();
  doReturn(Collections.singletonList(containerStatus1))
      .when(statusEvent1).getContainers();
   
  doReturn(completedContainerId2).when(containerStatus2).getContainerId();
  doReturn(Collections.singletonList(containerStatus2))
      .when(statusEvent2).getContainers();

  verify(scheduler,times(1)).handle(any(NodeUpdateSchedulerEvent.class)); 
  node.handle(statusEvent1);
  node.handle(statusEvent2);
  verify(scheduler,times(1)).handle(any(NodeUpdateSchedulerEvent.class));
  Assert.assertEquals(2, node.getQueueSize());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.EXPIRE));
  Assert.assertEquals(0, node.getQueueSize());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test(timeout=20000)
public void testUpdateHeartbeatResponseForCleanup() {
  RMNodeImpl node = getRunningNode();
  NodeId nodeId = node.getNodeID();

  // Expire a container
ContainerId completedContainerId = BuilderUtils.newContainerId(
		BuilderUtils.newApplicationAttemptId(
				BuilderUtils.newApplicationId(0, 0), 0), 0);
  node.handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId));
  Assert.assertEquals(1, node.getContainersToCleanUp().size());

  // Finish an application
  ApplicationId finishedAppId = BuilderUtils.newApplicationId(0, 1);
  node.handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
  Assert.assertEquals(1, node.getAppsToCleanup().size());

  // Verify status update does not clear containers/apps to cleanup
  // but updating heartbeat response for cleanup does
  RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent();
  node.handle(statusEvent);
  Assert.assertEquals(1, node.getContainersToCleanUp().size());
  Assert.assertEquals(1, node.getAppsToCleanup().size());
  NodeHeartbeatResponse hbrsp = Records.newRecord(NodeHeartbeatResponse.class);
  node.updateNodeHeartbeatResponseForCleanup(hbrsp);
  Assert.assertEquals(0, node.getContainersToCleanUp().size());
  Assert.assertEquals(0, node.getAppsToCleanup().size());
  Assert.assertEquals(1, hbrsp.getContainersToCleanup().size());
  Assert.assertEquals(completedContainerId, hbrsp.getContainersToCleanup().get(0));
  Assert.assertEquals(1, hbrsp.getApplicationsToCleanup().size());
  Assert.assertEquals(finishedAppId, hbrsp.getApplicationsToCleanup().get(0));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
private RMNodeImpl getUnhealthyNode() {
  RMNodeImpl node = getRunningNode();
  NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick",
      System.currentTimeMillis());
  node.handle(new RMNodeStatusEvent(node.getNodeID(), status,
      new ArrayList<ContainerStatus>(), null, null));
  Assert.assertEquals(NodeState.UNHEALTHY, node.getState());
  return node;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test
public void testNodesDefaultWithUnHealthyNode() throws JSONException,
    Exception {

  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1235", 5121);
  rm.sendNodeStarted(nm1);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);

  MockNM nm3 = rm.registerNode("h3:1236", 5122);
  rm.NMwaitForState(nm3.getNodeId(), NodeState.NEW);
  rm.sendNodeStarted(nm3);
  rm.NMwaitForState(nm3.getNodeId(), NodeState.RUNNING);
  RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
      .get(nm3.getNodeId());
  NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
      "test health report", System.currentTimeMillis());
  NodeStatus nodeStatus = NodeStatus.newInstance(nm3.getNodeId(), 1,
    new ArrayList<ContainerStatus>(), null, nodeHealth, null, null, null);
  node.handle(new RMNodeStatusEvent(nm3.getNodeId(), nodeStatus, null));
  rm.NMwaitForState(nm3.getNodeId(), NodeState.UNHEALTHY);

  ClientResponse response =
      r.path("ws").path("v1").path("cluster").path("nodes")
        .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  // 3 nodes, including the unhealthy node and the new node.
  assertEquals("incorrect number of elements", 3, nodeArray.length());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test
public void testNodesResourceUtilization() throws JSONException, Exception {
  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  rm.sendNodeStarted(nm1);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);

  RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
      .get(nm1.getNodeId());
  NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
      "test health report", System.currentTimeMillis());
  ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0,
      (float) 10.5);
  ResourceUtilization containerResource = ResourceUtilization.newInstance(
      2048, 0, (float) 5.05);
  NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
      new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource,
      nodeResource, null);
  node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, null));
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("nodes").accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  assertEquals("incorrect number of elements", 1, nodeArray.length());
  JSONObject info = nodeArray.getJSONObject(0);

  // verify the resource utilization
  verifyNodeInfo(info, nm1);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
private RMNodeStatusEvent getMockRMNodeStatusEventWithRunningApps() {
  NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);

  NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
  Boolean yes = new Boolean(true);
  doReturn(yes).when(healthStatus).getIsNodeHealthy();

  RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
  doReturn(healthStatus).when(event).getNodeHealthStatus();
  doReturn(response).when(event).getLatestResponse();
  doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
  doReturn(getAppIdList()).when(event).getKeepAliveAppIds();
  return event;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
private RMNodeStatusEvent getMockRMNodeStatusEventWithoutRunningApps() {
  NodeHeartbeatResponse response = mock(NodeHeartbeatResponse.class);

  NodeHealthStatus healthStatus = mock(NodeHealthStatus.class);
  Boolean yes = new Boolean(true);
  doReturn(yes).when(healthStatus).getIsNodeHealthy();

  RMNodeStatusEvent event = mock(RMNodeStatusEvent.class);
  doReturn(healthStatus).when(event).getNodeHealthStatus();
  doReturn(response).when(event).getLatestResponse();
  doReturn(RMNodeEventType.STATUS_UPDATE).when(event).getType();
  doReturn(null).when(event).getKeepAliveAppIds();
  return event;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test (timeout = 5000)
public void testExpiredContainer() {
  // Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  verify(scheduler).handle(any(NodeAddedSchedulerEvent.class));
  
  // Expire a container
  ContainerId completedContainerId = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  node.handle(new RMNodeCleanContainerEvent(null, completedContainerId));
  Assert.assertEquals(1, node.getContainersToCleanUp().size());
  
  // Now verify that scheduler isn't notified of an expired container
  // by checking number of 'completedContainers' it got in the previous event
  RMNodeStatusEvent statusEvent = getMockRMNodeStatusEvent(null);
  ContainerStatus containerStatus = mock(ContainerStatus.class);
  doReturn(completedContainerId).when(containerStatus).getContainerId();
  doReturn(Collections.singletonList(containerStatus)).
      when(statusEvent).getContainers();
  node.handle(statusEvent);
  /* Expect the scheduler call handle function 2 times
   * 1. RMNode status from new to Running, handle the add_node event
   * 2. handle the node update event
   */
  verify(scheduler,times(2)).handle(any(NodeUpdateSchedulerEvent.class));     
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent; //导入依赖的package包/类
@Test
public void testStatusUpdateOnDecommissioningNode() {
  RMNodeImpl node = getDecommissioningNode();
  ClusterMetrics cm = ClusterMetrics.getMetrics();
  int initialActive = cm.getNumActiveNMs();
  int initialDecommissioning = cm.getNumDecommissioningNMs();
  int initialDecommissioned = cm.getNumDecommisionedNMs();
  Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
  // Verify node in DECOMMISSIONING won't be changed by status update
  // with running apps
  RMNodeStatusEvent statusEvent = getMockRMNodeStatusEventWithRunningApps();
  node.handle(statusEvent);
  Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
  Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
  Assert.assertEquals("Decommissioning Nodes", initialDecommissioning,
      cm.getNumDecommissioningNMs());
  Assert.assertEquals("Decommissioned Nodes", initialDecommissioned,
      cm.getNumDecommisionedNMs());

  // Verify node in DECOMMISSIONING will be changed by status update
  // without running apps
  statusEvent = getMockRMNodeStatusEventWithoutRunningApps();
  node.handle(statusEvent);
  Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
  Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
  Assert.assertEquals("Decommissioning Nodes", initialDecommissioning - 1,
      cm.getNumDecommissioningNMs());
  Assert.assertEquals("Decommissioned Nodes", initialDecommissioned + 1,
      cm.getNumDecommisionedNMs());
}