Java NMClient.createNMClient方法代码示例

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public SolrMaster(CommandLine cli) throws Exception {
  this.cli = cli;
  Configuration hadoopConf = new Configuration();
  if (cli.hasOption("conf")) {
    hadoopConf.addResource(new Path(cli.getOptionValue("conf")));
    hadoopConf.reloadConfiguration();
  }
  conf = new YarnConfiguration(hadoopConf);

  nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  numContainersToWaitFor = Integer.parseInt(cli.getOptionValue("nodes"));
  memory = Integer.parseInt(cli.getOptionValue("memory", "512"));
  port = Integer.parseInt(cli.getOptionValue("port"));
  nextPort = port;

  SecureRandom random = new SecureRandom();
  this.randomStopKey = new BigInteger(130, random).toString(32);

  this.inetAddresses = getMyInetAddresses();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public StormAMRMClient(ApplicationAttemptId appAttemptId,
                       @SuppressWarnings("rawtypes") Map storm_conf,
                       YarnConfiguration hadoopConf) {
  this.appAttemptId = appAttemptId;
  this.storm_conf = storm_conf;
  this.hadoopConf = hadoopConf;
  Integer pri = Utils.getInt(storm_conf.get(Config.MASTER_CONTAINER_PRIORITY));
  this.DEFAULT_PRIORITY.setPriority(pri);
  this.containers = new TreeSet<Container>();
  numSupervisors = new AtomicInteger(0);

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  nmClient.init(hadoopConf);
  nmClient.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testAMRMClientWithContainerResourceChange()
    throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.createAMRMClient();
    Assert.assertNotNull(amClient);
    // asserting we are using the singleton instance cache
    Assert.assertSame(
        NMTokenCache.getSingleton(), amClient.getNMTokenCache());
    amClient.init(conf);
    amClient.start();
    assertEquals(STATE.STARTED, amClient.getServiceState());
    // start am nm client
    NMClientImpl nmClient = (NMClientImpl) NMClient.createNMClient();
    Assert.assertNotNull(nmClient);
    // asserting we are using the singleton instance cache
    Assert.assertSame(
        NMTokenCache.getSingleton(), nmClient.getNMTokenCache());
    nmClient.init(conf);
    nmClient.start();
    assertEquals(STATE.STARTED, nmClient.getServiceState());
    // am rm client register the application master with RM
    amClient.registerApplicationMaster("Host", 10000, "");
    // allocate three containers and make sure they are in RUNNING state
    List<Container> containers =
        allocateAndStartContainers(amClient, nmClient, 3);
    // perform container resource increase and decrease tests
    doContainerResourceChange(amClient, containers);
    // unregister and finish up the test
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
@Override
public void launch(int n, List<String> command, List<String> hosts,
    boolean verbose) throws Exception {
  final List<String> chmod = new ArrayList<String>();
  chmod.add("chmod");
  chmod.add("a+rx");
  chmod.add(System.getenv(Environment.LOG_DIRS.name()));
  final ProcessBuilder pb = new ProcessBuilder(chmod);
  pb.redirectOutput(Redirect.INHERIT);
  pb.redirectError(Redirect.INHERIT);
  pb.start();
  redirect(command); // TODO clone before mutating the command
  rmClient.init(conf);
  rmClient.start();
  final NMClient nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  rmClient.registerApplicationMaster("", 0, "");
  for (int i = 0; i < n; ++i) {
    final ContainerRequest request = new ContainerRequest(
        Resource.newInstance(256, 1), null, null, Priority.newInstance(0));
    rmClient.addContainerRequest(request);
  }
  int responseId = 0;
  for (int containers = 0; containers < n;) {
    final AllocateResponse response = rmClient.allocate(responseId++);
    for (final Container container : response.getAllocatedContainers()) {
      final ContainerLaunchContext ctx = ContainerLaunchContext
          .newInstance(null, null, command, null, null, null);
      nmClient.startContainer(container, ctx);
    }
    containers += response.getAllocatedContainers().size();
    try {
      Thread.sleep(100);
    } catch (final InterruptedException e) {
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public ApplicationMaster() {
    this.allocatedContainerMap = new HashMap<String, Container>();
    this.sendQueue = new ArrayBlockingQueue<JSONObject>(512);
    this.configuration = new YarnConfiguration();
    this.defaultLocalRes = new HashMap<String, LocalResource>();
    this.nmClient = NMClient.createNMClient();
    this.nmClient.init(this.configuration);
    this.nmClient.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public void start() {
    if (client != null) {
        return;
    }

    client = NMClient.createNMClient("Elasticsearch-YARN");
    YarnCompat.setNMTokenCache(client, tokenCache);
    client.init(cfg);
    client.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public YarnFlinkResourceManager(
	Configuration flinkConfig,
	YarnConfiguration yarnConfig,
	LeaderRetrievalService leaderRetrievalService,
	String applicationMasterHostName,
	String webInterfaceURL,
	ContaineredTaskManagerParameters taskManagerParameters,
	ContainerLaunchContext taskManagerLaunchContext,
	int yarnHeartbeatIntervalMillis,
	int maxFailedContainers,
	int numInitialTaskManagers,
	YarnResourceManagerCallbackHandler callbackHandler) {

	this(
		flinkConfig,
		yarnConfig,
		leaderRetrievalService,
		applicationMasterHostName,
		webInterfaceURL,
		taskManagerParameters,
		taskManagerLaunchContext,
		yarnHeartbeatIntervalMillis,
		maxFailedContainers,
		numInitialTaskManagers,
		callbackHandler,
		AMRMClientAsync.createAMRMClientAsync(yarnHeartbeatIntervalMillis, callbackHandler),
		NMClient.createNMClient());
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public StormAMRMClient(ApplicationAttemptId appAttemptID, @SuppressWarnings("rawtypes") Map storm_conf,
		YarnConfiguration hadoopConf) {
	this.appAttemptId = appAttemptID;
    this.storm_conf = storm_conf;
    this.hadoopConf = hadoopConf;
    Integer pri = Utils.getInt(storm_conf.get(Config.MASTER_CONTAINER_PRIORITY));
    this.DEFAULT_PRIORITY.setPriority(pri);
    this.containers = new TreeSet<Container>();
    numSupervisors = new AtomicInteger(0);

    // start am nm client
    nmClient = (NMClientImpl) NMClient.createNMClient();
    nmClient.init(hadoopConf);
    nmClient.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  LOG.error("command: " + this.command);
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();

  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();

  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl);


  // Ask RM to give us a bunch of containers
  for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  }
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster( FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();
  
  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();
  
  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
      appMasterTrackingUrl);

  Log.info("total container count: "+Integer.toString(totalContainerCount));
  
  // Ask RM to give us a bunch of containers
  //for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  //}
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public ApplicationMasterAsync(String command, int numContainersToWaitFor) {
    this.command = command;
    configuration = new YarnConfiguration();
    this.numContainersToWaitFor = numContainersToWaitFor;
    nmClient = NMClient.createNMClient();
    nmClient.init(configuration);
    nmClient.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public Hadoop21YarnNMClient(Configuration configuration) {
  this.nmClient = NMClient.createNMClient();
  nmClient.init(configuration);
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

    String command = args[0];

    int argi = 0;

    for (argi=1; argi<args.length - 1; argi++) {
      command += " " + args[argi];
    }

    final int n = Integer.valueOf(args[argi]);

    // Initialize clients to ResourceManager and NodeManagers
    Configuration conf = new YarnConfiguration();

    AMRMClient<ContainerRequest> rmClient = AMRMClient.createAMRMClient();
    rmClient.init(conf);
    rmClient.start();

    NMClient nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();

    // Register with ResourceManager
    rmClient.registerApplicationMaster("", 0, "");
    
    // Priority for worker containers - priorities are intra-application
    Priority priority = Records.newRecord(Priority.class);
    priority.setPriority(0);

    // Resource requirements for worker containers
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(128);
    capability.setVirtualCores(1);

    // Make container requests to ResourceManager
    for (int i=1; i <= n; i++) {
      ContainerRequest containerAsk = new ContainerRequest(capability, null, null, priority);
      System.out.println("=> Container Request " + i);
      rmClient.addContainerRequest(containerAsk);
    }

    // Obtain allocated containers and launch 
    int allocatedContainers = 0;
    while (allocatedContainers < n) {
      AllocateResponse response = rmClient.allocate(0);
      for (Container container : response.getAllocatedContainers()) {
        ++allocatedContainers;

        // Launch container by create ContainerLaunchContext
        ContainerLaunchContext ctx = 
            Records.newRecord(ContainerLaunchContext.class);
        ctx.setCommands(Collections.singletonList(
                command + 
                " >" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdouterr_am" + 
                " 2>&1"
                ));
        System.out.println("=> Launching Container " + allocatedContainers);
        nmClient.startContainer(container, ctx);
      }
      Thread.sleep(100);
    }

    // Now work with containers
    for (int container = 0; container < n; container++) {
      rmClient.allocate(0);
    }

    // Un-register with ResourceManager
    rmClient.unregisterApplicationMaster(
        FinalApplicationStatus.SUCCEEDED, "", "");
  }