Java AMRMClient.ContainerRequest方法代码示例

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
private void setupAndAddContainer(ContainerAllocatorEvent event) {
  LOG.info("submit container request to AM");
  Priority pri = Priority.newInstance(event.getPriority());
  Resource capability = event.getCapability();
  String[] nodes = new String[1];
  nodes[0] = event.getHostname();
  if (maxMem < capability.getMemory() |
      maxVcores < capability.getVirtualCores()) {
    LOG.warn("Exceeds cluster max resource for " + event.getId() + ", exiting.");
    eventHandler.handle(new JobEvent(context.getJob().getID(),JobEventType.JOB_ERROR));
    return;
  }
  AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability,nodes,null,pri);
  amRMClientAsync.addContainerRequest(request);
  pendingTasks.add(event.getId());
  resourceRecords.put(event.getId(),
      new ResourceRecord(request, event, event.getId(), capability.getMemory(), capability.getVirtualCores()));
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public TestingYarnFlinkResourceManager(
	Configuration flinkConfig,
	YarnConfiguration yarnConfig,
	LeaderRetrievalService leaderRetrievalService,
	String applicationMasterHostName,
	String webInterfaceURL,
	ContaineredTaskManagerParameters taskManagerParameters,
	ContainerLaunchContext taskManagerLaunchContext,
	int yarnHeartbeatIntervalMillis,
	int maxFailedContainers,
	int numInitialTaskManagers,
	YarnResourceManagerCallbackHandler callbackHandler,
	AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient,
	NMClient nodeManagerClient) {
	super(flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(params);
  LOG.info("Init CTXT: " + ctxt.getLocalResources());
  LOG.info("Init CTXT: " + ctxt.getCommands());
  
  this.resource = factory.createResource(params);
  this.priority = factory.createPriority(params.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = params.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(parameters);
  this.resource = factory.createResource(parameters);
  this.priority = factory.createPriority(parameters.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = parameters.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
  totalRequested.addAndGet(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
/**
 * @throws Exception If failed.
 */
public void testContainerAllocate() throws Exception {
    appMaster.setRmClient(rmMock);
    appMaster.setNmClient(new NMMock());

    props.cpusPerNode(2);
    props.memoryPerNode(1024);
    props.instances(3);

    Thread thread = runAppMaster(appMaster);

    List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 2, 1000);

    interruptedThread(thread);

    assertEquals(3, contRequests.size());

    for (AMRMClient.ContainerRequest req : contRequests) {
        assertEquals(2, req.getCapability().getVirtualCores());
        assertEquals(1024, req.getCapability().getMemory());
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
/**
 * Tests whether memory overhead is allocated within container memory.
 *
 * @throws Exception If failed.
 */
public void testMemoryOverHeadAllocation() throws Exception {
    appMaster.setRmClient(rmMock);
    appMaster.setNmClient(new NMMock());

    props.cpusPerNode(2);
    props.memoryPerNode(1024);
    props.memoryOverHeadPerNode(512);
    props.instances(3);

    Thread thread = runAppMaster(appMaster);

    List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);

    interruptedThread(thread);

    assertEquals(3, contRequests.size());

    for (AMRMClient.ContainerRequest req : contRequests) {
        assertEquals(2, req.getCapability().getVirtualCores());
        assertEquals(1024 + 512, req.getCapability().getMemory());
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
/**
* Tests whether memory overhead prevents from allocating container.
*
* @throws Exception If failed.
*/
public void testMemoryOverHeadPreventAllocation() throws Exception {
   rmMock.availableRes(new MockResource(1024, 2));
   appMaster.setRmClient(rmMock);
   appMaster.setNmClient(new NMMock());

   props.cpusPerNode(2);
   props.memoryPerNode(1024);
   props.memoryOverHeadPerNode(512);
   props.instances(3);

   Thread thread = runAppMaster(appMaster);

   List<AMRMClient.ContainerRequest> contRequests = collectRequests(rmMock, 1, 1000);

   interruptedThread(thread);

   assertEquals(0, contRequests.size());
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
/**
 * Cancel just one of the priority levels
 * @param priority priority to cancel
 * @param count count to cancel
 * @return number of requests cancelled
 */
protected int cancelSinglePriorityRequests(Priority priority,
    int count) {
  List<Collection<AMRMClient.ContainerRequest>> requestSets =
      client.getMatchingRequests(priority, "", maxResources);
  if (count <= 0) {
    return 0;
  }
  int remaining = count;
  for (Collection<AMRMClient.ContainerRequest> requestSet : requestSets) {
    if (remaining == 0) {
      break;
    }
    for (AMRMClient.ContainerRequest request : requestSet) {
      if (remaining == 0) {
        break;
      }
      // a single release
      client.removeContainerRequest(request);
      remaining --;
    }
  }
  return remaining;
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
private int getAllocatedContainersNumber(
    AMRMClientImpl<AMRMClient.ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
    int allocatedContainerCount = 0;
    while (iterationsLeft-- > 0) {
        Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
        AllocateResponse allocResponse = amClient.allocate(0.1f);
        assertTrue(amClient.ask.size() == 0);
        assertTrue(amClient.release.size() == 0);

        assertTrue(NODECOUNT == amClient.getClusterNodeCount());
        allocatedContainerCount += allocResponse.getAllocatedContainers().size();

        if(allocatedContainerCount == 0) {
            // sleep to let NM's heartbeat to RM and trigger allocations
            sleep(SpliceTestYarnPlatform.DEFAULT_HEARTBEAT_INTERVAL);
        }
    }
    return allocatedContainerCount;
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)  @Ignore("Broken by dependency change")
public void testAMRMClient() throws YarnException, IOException {
    AMRMClient<AMRMClient.ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = AMRMClient.createAMRMClient();
        amClient.init(testYarnParticipant.getConfig());
        amClient.start();

        amClient.registerApplicationMaster("Host", 10000, "");

        testAllocation((AMRMClientImpl<AMRMClient.ContainerRequest>)amClient);

        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
                                             null, null);

    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.stop();
        }
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Override
public void addSupervisors(int number, int containerMemory, int containerVcores) throws TException {
    if (containerMemory > jstormMasterContext.maxMemory) {
        containerMemory = jstormMasterContext.maxMemory;
    }
    if (containerVcores > jstormMasterContext.maxVcores) {
        containerVcores = jstormMasterContext.maxVcores;
    }
    for (int i = 0; i < number; i++) {
        //set priority to 0 which identity this container is allocated for supervisor
        AMRMClient.ContainerRequest containerAsk = jstormMaster.setupContainerAskForRM(containerMemory, containerVcores, 0, "*");
        try {
            requestQueue.put(containerAsk);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        amRMClient.addContainerRequest(containerAsk);
        jstormMasterContext.numRequestedContainers.getAndIncrement();
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Override
public void addSpecSupervisor(int number, int container_memory, int container_vcorecount, List<String> racks, List<String> hosts) throws TException {

    LOG.info("number:" + number + "; memory:" + container_memory + "; vcore:" + container_vcorecount + "; racks:" + JstormYarnUtils.join(racks, ",", false));
    LOG.info("hosts:" + JstormYarnUtils.join(hosts, ",", false));

    if (container_memory > jstormMasterContext.maxMemory) {
        container_vcorecount = jstormMasterContext.maxMemory;
    }
    if (container_vcorecount > jstormMasterContext.maxVcores) {
        container_vcorecount = jstormMasterContext.maxVcores;
    }
    for (int i = 0; i < number; i++) {
        AMRMClient.ContainerRequest containerAsk = jstormMaster.setupContainerAskForRM(container_memory, container_vcorecount, 0, racks.toArray(new String[0]), hosts.toArray(new String[0]));
        try {
            requestQueue.put(containerAsk);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        amRMClient.addContainerRequest(containerAsk);
        jstormMasterContext.numRequestedContainers.getAndIncrement();
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Override
public void startSpecNimbus(int number, int containerMemory, int containerVirtualCores, List<String> racks, List<String> hosts) throws TException {
    //set priority to 1 which identity this container is allocated for nimbus
    String dstHost = "*";
    if (!jstormMasterContext.previousNimbusHost.equals("") && jstormMasterContext.nimbusContainers.size() == 0)
        dstHost = jstormMasterContext.previousNimbusHost;

    LOG.info("dstHost:" + dstHost);
    AMRMClient.ContainerRequest containerAsk = jstormMaster.setupContainerAskForRM(containerMemory, containerVirtualCores, 1,
            racks.toArray(new String[0]), hosts.toArray(new String[0]));
    try {
        requestQueue.put(containerAsk);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    amRMClient.addContainerRequest(containerAsk);
    jstormMasterContext.numRequestedContainers.getAndIncrement();
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Override
public void startNimbus(int number, int containerMemory, int containerVirtualCores) throws TException {
    //set priority to 1 which identity this container is allocated for nimbus
    String dstHost = "*";
    if (!jstormMasterContext.previousNimbusHost.equals("") && jstormMasterContext.nimbusContainers.size() == 0)
        dstHost = jstormMasterContext.previousNimbusHost;

    LOG.info("dstHost:" + dstHost);
    AMRMClient.ContainerRequest containerAsk = jstormMaster.setupContainerAskForRM(containerMemory, containerVirtualCores, 1, dstHost);
    try {
        requestQueue.put(containerAsk);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    amRMClient.addContainerRequest(containerAsk);
    jstormMasterContext.numRequestedContainers.getAndIncrement();
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public synchronized void removeContainerRequest(AMRMClient.ContainerRequest request) throws Exception {

    ResourceDescriptor resource = ResourceDescriptor.fromResource(request.getCapability());
    List<String> nodeList = request.getNodes();

    for (String nodeName : nodeList) {
      if (!candidateRequestNodes.containsKey(nodeName)) {
        LOGGER.logWarning(
            "removeContainerRequest: node is not exist: " + nodeName);
        continue;
      }
      candidateRequestNodes.get(nodeName).removeContainerRequest(resource);
      return;
    }
  }