Java NMClient.start方法代码示例

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
@Override
public void launch(int n, List<String> command, List<String> hosts,
    boolean verbose) throws Exception {
  final List<String> chmod = new ArrayList<String>();
  chmod.add("chmod");
  chmod.add("a+rx");
  chmod.add(System.getenv(Environment.LOG_DIRS.name()));
  final ProcessBuilder pb = new ProcessBuilder(chmod);
  pb.redirectOutput(Redirect.INHERIT);
  pb.redirectError(Redirect.INHERIT);
  pb.start();
  redirect(command); // TODO clone before mutating the command
  rmClient.init(conf);
  rmClient.start();
  final NMClient nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  rmClient.registerApplicationMaster("", 0, "");
  for (int i = 0; i < n; ++i) {
    final ContainerRequest request = new ContainerRequest(
        Resource.newInstance(256, 1), null, null, Priority.newInstance(0));
    rmClient.addContainerRequest(request);
  }
  int responseId = 0;
  for (int containers = 0; containers < n;) {
    final AllocateResponse response = rmClient.allocate(responseId++);
    for (final Container container : response.getAllocatedContainers()) {
      final ContainerLaunchContext ctx = ContainerLaunchContext
          .newInstance(null, null, command, null, null, null);
      nmClient.startContainer(container, ctx);
    }
    containers += response.getAllocatedContainers().size();
    try {
      Thread.sleep(100);
    } catch (final InterruptedException e) {
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

    String command = args[0];

    int argi = 0;

    for (argi=1; argi<args.length - 1; argi++) {
      command += " " + args[argi];
    }

    final int n = Integer.valueOf(args[argi]);

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    rmClient.registerApplicationMaster("", 0, "");
    
    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i=1; i <= n; i++) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      System.out.println("=> Container Request " + i);
      rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers and launch 
    int allocatedContainers = 0;
    while (allocatedContainers < n) {
      AllocateResponse response = rmClient.allocate(0);
      for (Container container : response.getAllocatedContainers()) {
        ++allocatedContainers;

        // Launch container by create ContainerLaunchContext
        ContainerLaunchContext ctx = 
            Records.newRecord(ContainerLaunchContext.class);
        ctx.setCommands(Collections.singletonList(
                command + 
                " >" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdouterr_am" + 
                " 2>&1"
                ));
        System.out.println("=> Launching Container " + allocatedContainers);
        nmClient.startContainer(container, ctx);
      }
      Thread.sleep(100);
    }

    // Now work with containers
    for (int container = 0; container < n; container++) {
      rmClient.allocate(0);
    }

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(
        FinalApplicationStatus.SUCCEEDED, "", "");
  }
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    
    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    System.out.println("registerApplicationMaster 0");
    rmClient.registerApplicationMaster("", 0, "");
    System.out.println("registerApplicationMaster 1");
    
    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i = 0; i < n; ++i) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      System.out.println("Making res-req " + i);
      rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers, launch and check for responses
    int responseId = 0;
    int completedContainers = 0;
    while (completedContainers < n) {
        AllocateResponse response = rmClient.allocate(responseId++);
        for (Container container : response.getAllocatedContainers()) {
            // Launch container by create ContainerLaunchContext
            ContainerLaunchContext ctx =
                    Records.newRecord(ContainerLaunchContext.class);
            ctx.setCommands(
                    Collections.singletonList(
                            command +
                                    " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                                    " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
                    ));
            System.out.println("Launching container " + container.getId());
            nmClient.startContainer(container, ctx);
        }
        for (ContainerStatus status : response.getCompletedContainersStatuses()) {
            ++completedContainers;
            System.out.println("Completed container " + status.getContainerId());
        }
        Thread.sleep(100);
    }

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(
        FinalApplicationStatus.SUCCEEDED, "", "");
  }