Java AMRMClient类代码示例

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
private void setupAndAddContainer(ContainerAllocatorEvent event) {
  LOG.info("submit container request to AM");
  Priority pri = Priority.newInstance(event.getPriority());
  Resource capability = event.getCapability();
  String[] nodes = new String[1];
  nodes[0] = event.getHostname();
  if (maxMem < capability.getMemory() |
      maxVcores < capability.getVirtualCores()) {
    LOG.warn("Exceeds cluster max resource for " + event.getId() + ", exiting.");
    eventHandler.handle(new JobEvent(context.getJob().getID(),JobEventType.JOB_ERROR));
    return;
  }
  AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability,nodes,null,pri);
  amRMClientAsync.addContainerRequest(request);
  pendingTasks.add(event.getId());
  resourceRecords.put(event.getId(),
      new ResourceRecord(request, event, event.getId(), capability.getMemory(), capability.getVirtualCores()));
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
public TestingYarnFlinkResourceManager(
	Configuration flinkConfig,
	YarnConfiguration yarnConfig,
	LeaderRetrievalService leaderRetrievalService,
	String applicationMasterHostName,
	String webInterfaceURL,
	ContaineredTaskManagerParameters taskManagerParameters,
	ContainerLaunchContext taskManagerLaunchContext,
	int yarnHeartbeatIntervalMillis,
	int maxFailedContainers,
	int numInitialTaskManagers,
	YarnResourceManagerCallbackHandler callbackHandler,
	AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient,
	NMClient nodeManagerClient) {
	super(flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
/**
 * Issue requests to AM RM Client again if previous container requests expired and were not allocated by Yarn
 * @param amRmClient
 * @param requestedResources
 * @param loopCounter
 * @param resourceRequestor
 * @param containerRequests
 * @param removedContainerRequests
 */
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests)
{
  if (!requestedResources.isEmpty()) {
    for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
      /*
       * Create container requests again if pending requests were not allocated by Yarn till timeout.
       */
      if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
        StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
        LOG.debug("Request for container {} timed out. Re-requesting container", csr.container);
        removedContainerRequests.add(entry.getValue().getRight());
        ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
        entry.getValue().setLeft(loopCounter);
        entry.getValue().setRight(cr);
        containerRequests.add(cr);
      }
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
private void requestContainers(int containersRequested) throws IOException, YarnException {
  for (int i = 0; i < containersRequested; i++) {
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    Resource capability = Records.newRecord(Resource.class);
    int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
    capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ?
        this.requestedContainerMemoryMbs : maxMemoryCapacity);
    int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
    capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ?
        this.requestedContainerCores : maxCoreCapacity);

    this.amrmClientAsync.addContainerRequest(new AMRMClient.ContainerRequest(capability, null, null, priority));
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(params);
  LOG.info("Init CTXT: " + ctxt.getLocalResources());
  LOG.info("Init CTXT: " + ctxt.getCommands());
  
  this.resource = factory.createResource(params);
  this.priority = factory.createPriority(params.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = params.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(parameters);
  this.resource = factory.createResource(parameters);
  this.priority = factory.createPriority(parameters.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = parameters.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
  totalRequested.addAndGet(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();
    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  final AllocateResponse shutDownResponse = createAllocateResponse(
      new ArrayList<ContainerStatus>(), new ArrayList<Container>(), null);
  shutDownResponse.setAMCommand(AMCommand.AM_SHUTDOWN);
  when(client.allocate(anyFloat())).thenReturn(shutDownResponse);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
/**
 * @throws Exception If failed.
 */
public void testContainerAllocate() throws Exception {
    appMaster.setRmClient(rmMock);
    appMaster.setNmClient(new NMMock());

    props.cpusPerNode(2);
    props.memoryPerNode(1024);
    props.instances(3);

    Thread thread = runAppMaster(appMaster);

    List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 2, 1000);

    interruptedThread(thread);

    assertEquals(3, contRequests.size());

    for (AMRMClient.ContainerRequest req : contRequests) {
        assertEquals(2, req.getCapability().getVirtualCores());
        assertEquals(1024, req.getCapability().getMemory());
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
/**
 * Tests whether memory overhead is allocated within container memory.
 *
 * @throws Exception If failed.
 */
public void testMemoryOverHeadAllocation() throws Exception {
    appMaster.setRmClient(rmMock);
    appMaster.setNmClient(new NMMock());

    props.cpusPerNode(2);
    props.memoryPerNode(1024);
    props.memoryOverHeadPerNode(512);
    props.instances(3);

    Thread thread = runAppMaster(appMaster);

    List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);

    interruptedThread(thread);

    assertEquals(3, contRequests.size());

    for (AMRMClient.ContainerRequest req : contRequests) {
        assertEquals(2, req.getCapability().getVirtualCores());
        assertEquals(1024 + 512, req.getCapability().getMemory());
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
/**
* Tests whether memory overhead prevents from allocating container.
*
* @throws Exception If failed.
*/
public void testMemoryOverHeadPreventAllocation() throws Exception {
   rmMock.availableRes(new MockResource(1024, 2));
   appMaster.setRmClient(rmMock);
   appMaster.setNmClient(new NMMock());

   props.cpusPerNode(2);
   props.memoryPerNode(1024);
   props.memoryOverHeadPerNode(512);
   props.instances(3);

   Thread thread = runAppMaster(appMaster);

   List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);

   interruptedThread(thread);

   assertEquals(0, contRequests.size());
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
private void requestContainer(Optional<String> preferredNode) {
  Priority priority = Records.newRecord(Priority.class);
  priority.setPriority(0);

  Resource capability = Records.newRecord(Resource.class);
  int maxMemoryCapacity = this.maxResourceCapacity.get().getMemory();
  capability.setMemory(this.requestedContainerMemoryMbs <= maxMemoryCapacity ?
      this.requestedContainerMemoryMbs : maxMemoryCapacity);
  int maxCoreCapacity = this.maxResourceCapacity.get().getVirtualCores();
  capability.setVirtualCores(this.requestedContainerCores <= maxCoreCapacity ?
      this.requestedContainerCores : maxCoreCapacity);

  String[] preferredNodes = preferredNode.isPresent() ? new String[] {preferredNode.get()} : null;
  this.amrmClientAsync.addContainerRequest(
      new AMRMClient.ContainerRequest(capability, preferredNodes, null, priority));
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入依赖的package包/类
/**
 * Cancel just one of the priority levels
 * @param priority priority to cancel
 * @param count count to cancel
 * @return number of requests cancelled
 */
protected int cancelSinglePriorityRequests(Priority priority,
    int count) {
  List<Collection<AMRMClient.ContainerRequest>> requestSets =
      client.getMatchingRequests(priority, "", maxResources);
  if (count <= 0) {
    return 0;
  }
  int remaining = count;
  for (Collection<AMRMClient.ContainerRequest> requestSet : requestSets) {
    if (remaining == 0) {
      break;
    }
    for (AMRMClient.ContainerRequest request : requestSet) {
      if (remaining == 0) {
        break;
      }
      // a single release
      client.removeContainerRequest(request);
      remaining --;
    }
  }
  return remaining;
}