Java NMClient类代码示例

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
public TestingYarnFlinkResourceManager(
	Configuration flinkConfig,
	YarnConfiguration yarnConfig,
	LeaderRetrievalService leaderRetrievalService,
	String applicationMasterHostName,
	String webInterfaceURL,
	ContaineredTaskManagerParameters taskManagerParameters,
	ContainerLaunchContext taskManagerLaunchContext,
	int yarnHeartbeatIntervalMillis,
	int maxFailedContainers,
	int numInitialTaskManagers,
	YarnResourceManagerCallbackHandler callbackHandler,
	AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient,
	NMClient nodeManagerClient) {
	super(flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient);
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
public SolrMaster(CommandLine cli) throws Exception {
  this.cli = cli;
  Configuration hadoopConf = new Configuration();
  if (cli.hasOption("conf")) {
    hadoopConf.addResource(new Path(cli.getOptionValue("conf")));
    hadoopConf.reloadConfiguration();
  }
  conf = new YarnConfiguration(hadoopConf);

  nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  numContainersToWaitFor = Integer.parseInt(cli.getOptionValue("nodes"));
  memory = Integer.parseInt(cli.getOptionValue("memory", "512"));
  port = Integer.parseInt(cli.getOptionValue("port"));
  nextPort = port;

  SecureRandom random = new SecureRandom();
  this.randomStopKey = new BigInteger(130, random).toString(32);

  this.inetAddresses = getMyInetAddresses();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
public StormAMRMClient(ApplicationAttemptId appAttemptId,
                       @SuppressWarnings("rawtypes") Map storm_conf,
                       YarnConfiguration hadoopConf) {
  this.appAttemptId = appAttemptId;
  this.storm_conf = storm_conf;
  this.hadoopConf = hadoopConf;
  Integer pri = Utils.getInt(storm_conf.get(Config.MASTER_CONTAINER_PRIORITY));
  this.DEFAULT_PRIORITY.setPriority(pri);
  this.containers = new TreeSet<Container>();
  numSupervisors = new AtomicInteger(0);

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  nmClient.init(hadoopConf);
  nmClient.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name, client, callbackHandler);
  this.client = client;
  this.callbackHandler = callbackHandler;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name);
  this.setClient(client);
  this.setCallbackHandler(callbackHandler);
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
    AbstractCallbackHandler callbackHandler) {
  super(name, client, callbackHandler);
  this.client = client;
  this.callbackHandler = callbackHandler;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
@Private
@VisibleForTesting
@Deprecated
protected NMClientAsync(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name);
  this.setClient(client);
  this.setCallbackHandler(callbackHandler);
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
@Test(timeout=60000)
public void testAMRMClientWithContainerResourceChange()
    throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.createAMRMClient();
    Assert.assertNotNull(amClient);
    // asserting we are using the singleton instance cache
    Assert.assertSame(
        NMTokenCache.getSingleton(), amClient.getNMTokenCache());
    amClient.init(conf);
    amClient.start();
    assertEquals(STATE.STARTED, amClient.getServiceState());
    // start am nm client
    NMClientImpl nmClient = (NMClientImpl) NMClient.createNMClient();
    Assert.assertNotNull(nmClient);
    // asserting we are using the singleton instance cache
    Assert.assertSame(
        NMTokenCache.getSingleton(), nmClient.getNMTokenCache());
    nmClient.init(conf);
    nmClient.start();
    assertEquals(STATE.STARTED, nmClient.getServiceState());
    // am rm client register the application master with RM
    amClient.registerApplicationMaster("Host", 10000, "");
    // allocate three containers and make sure they are in RUNNING state
    List<Container> containers =
        allocateAndStartContainers(amClient, nmClient, 3);
    // perform container resource increase and decrease tests
    doContainerResourceChange(amClient, containers);
    // unregister and finish up the test
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).increaseContainerResource(
          any(Container.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Increase Resource Exception"))
          .when(client).increaseContainerResource(any(Container.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
@Override
public void launch(int n, List<String> command, List<String> hosts,
    boolean verbose) throws Exception {
  final List<String> chmod = new ArrayList<String>();
  chmod.add("chmod");
  chmod.add("a+rx");
  chmod.add(System.getenv(Environment.LOG_DIRS.name()));
  final ProcessBuilder pb = new ProcessBuilder(chmod);
  pb.redirectOutput(Redirect.INHERIT);
  pb.redirectError(Redirect.INHERIT);
  pb.start();
  redirect(command); // TODO clone before mutating the command
  rmClient.init(conf);
  rmClient.start();
  final NMClient nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  rmClient.registerApplicationMaster("", 0, "");
  for (int i = 0; i < n; ++i) {
    final ContainerRequest request = new ContainerRequest(
        Resource.newInstance(256, 1), null, null, Priority.newInstance(0));
    rmClient.addContainerRequest(request);
  }
  int responseId = 0;
  for (int containers = 0; containers < n;) {
    final AllocateResponse response = rmClient.allocate(responseId++);
    for (final Container container : response.getAllocatedContainers()) {
      final ContainerLaunchContext ctx = ContainerLaunchContext
          .newInstance(null, null, command, null, null, null);
      nmClient.startContainer(container, ctx);
    }
    containers += response.getAllocatedContainers().size();
    try {
      Thread.sleep(100);
    } catch (final InterruptedException e) {
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
public ApplicationMaster() {
    this.allocatedContainerMap = new HashMap<String, Container>();
    this.sendQueue = new ArrayBlockingQueue<JSONObject>(512);
    this.configuration = new YarnConfiguration();
    this.defaultLocalRes = new HashMap<String, LocalResource>();
    this.nmClient = NMClient.createNMClient();
    this.nmClient.init(this.configuration);
    this.nmClient.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
public void start() {
    if (client != null) {
        return;
    }

    client = NMClient.createNMClient("Elasticsearch-YARN");
    YarnCompat.setNMTokenCache(client, tokenCache);
    client.init(cfg);
    client.start();
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
 

import org.apache.hadoop.yarn.client.api.NMClient; //导入依赖的package包/类
public YarnFlinkResourceManager(
	Configuration flinkConfig,
	YarnConfiguration yarnConfig,
	LeaderRetrievalService leaderRetrievalService,
	String applicationMasterHostName,
	String webInterfaceURL,
	ContaineredTaskManagerParameters taskManagerParameters,
	ContainerLaunchContext taskManagerLaunchContext,
	int yarnHeartbeatIntervalMillis,
	int maxFailedContainers,
	int numInitialTaskManagers,
	YarnResourceManagerCallbackHandler callbackHandler) {

	this(
		flinkConfig,
		yarnConfig,
		leaderRetrievalService,
		applicationMasterHostName,
		webInterfaceURL,
		taskManagerParameters,
		taskManagerLaunchContext,
		yarnHeartbeatIntervalMillis,
		maxFailedContainers,
		numInitialTaskManagers,
		callbackHandler,
		AMRMClientAsync.createAMRMClientAsync(yarnHeartbeatIntervalMillis, callbackHandler),
		NMClient.createNMClient());
}