Java AMRMClient.addContainerRequest方法代码示例

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
    ContainerRequest request) {
  try {
    client.addContainerRequest(request);
  } catch (InvalidContainerRequestException e) {
    return;
  }
  Assert.fail();
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
private List<Container> allocateAndStartContainers(
    final AMRMClient<ContainerRequest> amClient, final NMClient nmClient,
    int num) throws YarnException, IOException {
  // set up allocation requests
  for (int i = 0; i < num; ++i) {
    amClient.addContainerRequest(
        new ContainerRequest(capability, nodes, racks, priority));
  }
  // send allocation requests
  amClient.allocate(0.1f);
  // sleep to let NM's heartbeat to RM and trigger allocations
  sleep(150);
  // get allocations
  AllocateResponse allocResponse = amClient.allocate(0.1f);
  List<Container> containers = allocResponse.getAllocatedContainers();
  Assert.assertEquals(num, containers.size());
  // build container launch context
  Credentials ts = new Credentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  ts.writeTokenStorageToStream(dob);
  ByteBuffer securityTokens =
      ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
  // start a process long enough for increase/decrease action to take effect
  ContainerLaunchContext clc = BuilderUtils.newContainerLaunchContext(
      Collections.<String, LocalResource>emptyMap(),
      new HashMap<String, String>(), Arrays.asList("sleep", "100"),
      new HashMap<String, ByteBuffer>(), securityTokens,
      new HashMap<ApplicationAccessType, String>());
  // start the containers and make sure they are in RUNNING state
  try {
    for (int i = 0; i < num; i++) {
      Container container = containers.get(i);
      nmClient.startContainer(container, clc);
      // NodeManager may still need some time to get the stable
      // container status
      while (true) {
        ContainerStatus status = nmClient.getContainerStatus(
            container.getId(), container.getNodeId());
        if (status.getState() == ContainerState.RUNNING) {
          break;
        }
        sleep(100);
      }
    }
  } catch (YarnException e) {
    throw new AssertionError("Exception is not expected: " + e);
  }
  // sleep to let NM's heartbeat to RM to confirm container launch
  sleep(200);
  return containers;
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

    String command = args[0];

    int argi = 0;

    for (argi=1; argi<args.length - 1; argi++) {
      command += " " + args[argi];
    }

    final int n = Integer.valueOf(args[argi]);

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    rmClient.registerApplicationMaster("", 0, "");
    
    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i=1; i <= n; i++) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      System.out.println("=> Container Request " + i);
      rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers and launch 
    int allocatedContainers = 0;
    while (allocatedContainers < n) {
      AllocateResponse response = rmClient.allocate(0);
      for (Container container : response.getAllocatedContainers()) {
        ++allocatedContainers;

        // Launch container by create ContainerLaunchContext
        ContainerLaunchContext ctx = 
            Records.newRecord(ContainerLaunchContext.class);
        ctx.setCommands(Collections.singletonList(
                command + 
                " >" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdouterr_am" + 
                " 2>&1"
                ));
        System.out.println("=> Launching Container " + allocatedContainers);
        nmClient.startContainer(container, ctx);
      }
      Thread.sleep(100);
    }

    // Now work with containers
    for (int container = 0; container < n; container++) {
      rmClient.allocate(0);
    }

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(
        FinalApplicationStatus.SUCCEEDED, "", "");
  }
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    
    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    System.out.println("registerApplicationMaster 0");
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("registerApplicationMaster 1");
    
    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i = 0; i < n; ++i) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      System.out.println("Making res-req " + i);
      rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    while (completedContainers < n) {
        AllocateResponse response = rmClient.allocate(responseId++);
        for (Container container : response.getAllocatedContainers()) {
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx =
                    Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(
                            command +
                                    " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                                    " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
                    ));
            System.out.println("Launching container " + container.getId());
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            System.out.println("Completed container " + status.getContainerId());
        }
        Thread.sleep(100);
    }

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(
        FinalApplicationStatus.SUCCEEDED, "", "");
  }