Java AMRMClient.createAMRMClient方法代码示例

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();
    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)  @Ignore("Broken by dependency change")
public void testAMRMClient() throws YarnException, IOException {
    AMRMClient<AMRMClient.ContainerRequest> amClient = null;
    try {
        // start am rm client
        amClient = AMRMClient.createAMRMClient();
        amClient.init(testYarnParticipant.getConfig());
        amClient.start();

        amClient.registerApplicationMaster("Host", 10000, "");

        testAllocation((AMRMClientImpl<AMRMClient.ContainerRequest>)amClient);

        amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
                                             null, null);

    } finally {
        if (amClient != null && amClient.getServiceState() == Service.STATE.STARTED) {
            amClient.stop();
        }
    }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test(timeout=60000)
public void testAMRMClientWithContainerResourceChange()
    throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.createAMRMClient();
    Assert.assertNotNull(amClient);
    // asserting we are using the singleton instance cache
    Assert.assertSame(
        NMTokenCache.getSingleton(), amClient.getNMTokenCache());
    amClient.init(conf);
    amClient.start();
    assertEquals(STATE.STARTED, amClient.getServiceState());
    // start am nm client
    NMClientImpl nmClient = (NMClientImpl) NMClient.createNMClient();
    Assert.assertNotNull(nmClient);
    // asserting we are using the singleton instance cache
    Assert.assertSame(
        NMTokenCache.getSingleton(), nmClient.getNMTokenCache());
    nmClient.init(conf);
    nmClient.start();
    assertEquals(STATE.STARTED, nmClient.getServiceState());
    // am rm client register the application master with RM
    amClient.registerApplicationMaster("Host", 10000, "");
    // allocate three containers and make sure they are in RUNNING state
    List<Container> containers =
        allocateAndStartContainers(amClient, nmClient, 3);
    // perform container resource increase and decrease tests
    doContainerResourceChange(amClient, containers);
    // unregister and finish up the test
    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public Hadoop21YarnAMClient(Configuration conf) {
  super(ApplicationConstants.Environment.CONTAINER_ID.name());

  this.amrmClient = AMRMClient.createAMRMClient();
  this.amrmClient.init(conf);
  this.nmClient = new Hadoop21YarnNMClient(conf);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
public void start() {
    if (client != null) {
        return;
    }

    client = AMRMClient.createAMRMClient();
    YarnCompat.setNMTokenCache(client, nmTokenCache);
    client.init(cfg);
    client.start();
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
private void registerAndAllocate() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(3, amClient.blacklistAdditions.size());
    assertEquals(0, amClient.blacklistRemovals.size());
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(1, amClient.blacklistAdditions.size());
    assertEquals(2, amClient.blacklistRemovals.size());
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(2, amClient.blacklistAdditions.size());
    assertEquals(1, amClient.blacklistRemovals.size());
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertEquals(3, amClient.ask.size());
    assertEquals(0, amClient.release.size());
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertEquals(0, allocatedContainerCount);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(2, allocatedContainerCount);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(1, amClient.blacklistAdditions.size());
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Override
protected void serviceInit(Configuration conf) throws Exception
{
  LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());

  FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
  try {
    this.dag = LogicalPlan.read(fis);
  } finally {
    fis.close();
  }
  // "debug" simply dumps all data using LOG.info
  if (dag.isDebug()) {
    dumpOutDebugInfo();
  }
  dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
  this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
  dag = this.dnmgr.getLogicalPlan();
  this.appContext = new ClusterAppContextImpl(dag.getAttributes());

  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
  StringCodecs.loadConverters(codecs);

  LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

  // Setup security configuration such as that for web security
  SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));

  if (UserGroupInformation.isSecurityEnabled()) {
    // TODO :- Need to perform token renewal
    delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
  }
  this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
  addService(nmClient);
  this.amRmClient = AMRMClient.createAMRMClient();
  addService(amRmClient);

  // start RPC server
  int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
  this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
  addService(heartbeatListener);

  AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
  if (appDataPushTransport != null) {
    this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
    addService(this.appDataPushAgent);
  }
  initApexPluginDispatcher();

  // Initialize all services added above
  super.serviceInit(conf);
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAllocationWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    
    assertTrue(amClient.ask.size() == 0);
    assertTrue(amClient.release.size() == 0);
    
    ContainerRequest storedContainer1 = 
        new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer1);
    assertTrue(amClient.ask.size() == 3);
    assertTrue(amClient.release.size() == 0);
    
    List<String> localNodeBlacklist = new ArrayList<String>();
    localNodeBlacklist.add(node);
    
    // put node in black list, so no container assignment
    amClient.updateBlacklist(localNodeBlacklist, null);

    int allocatedContainerCount = getAllocatedContainersNumber(amClient,
      DEFAULT_ITERATION);
    // the only node is in blacklist, so no allocation
    assertTrue(allocatedContainerCount == 0);

    // Remove node from blacklist, so get assigned with 2
    amClient.updateBlacklist(null, localNodeBlacklist);
    ContainerRequest storedContainer2 = 
            new ContainerRequest(capability, nodes, racks, priority);
    amClient.addContainerRequest(storedContainer2);
    allocatedContainerCount = getAllocatedContainersNumber(amClient,
        DEFAULT_ITERATION);
    assertEquals(allocatedContainerCount, 2);
    
    // Test in case exception in allocate(), blacklist is kept
    assertTrue(amClient.blacklistAdditions.isEmpty());
    assertTrue(amClient.blacklistRemovals.isEmpty());
    
    // create a invalid ContainerRequest - memory value is minus
    ContainerRequest invalidContainerRequest = 
        new ContainerRequest(Resource.newInstance(-1024, 1),
            nodes, racks, priority);
    amClient.addContainerRequest(invalidContainerRequest);
    amClient.updateBlacklist(localNodeBlacklist, null);
    try {
      // allocate() should complain as ContainerRequest is invalid.
      amClient.allocate(0.1f);
      fail("there should be an exception here.");
    } catch (Exception e) {
      assertEquals(amClient.blacklistAdditions.size(), 1);
    }
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.AMRMClient; //导入方法依赖的package包/类
@Test (timeout=60000)
public void testAMRMClientWithBlacklist() throws YarnException, IOException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
          .<ContainerRequest> createAMRMClient();
    amClient.init(conf);
    amClient.start();
    amClient.registerApplicationMaster("Host", 10000, "");
    String[] nodes = {"node1", "node2", "node3"};
    
    // Add nodes[0] and nodes[1]
    List<String> nodeList01 = new ArrayList<String>();
    nodeList01.add(nodes[0]);
    nodeList01.add(nodes[1]);
    amClient.updateBlacklist(nodeList01, null);
    assertEquals(amClient.blacklistAdditions.size(),2);
    assertEquals(amClient.blacklistRemovals.size(),0);
    
    // Add nodes[0] again, verify it is not added duplicated.
    List<String> nodeList02 = new ArrayList<String>();
    nodeList02.add(nodes[0]);
    nodeList02.add(nodes[2]);
    amClient.updateBlacklist(nodeList02, null);
    assertEquals(amClient.blacklistAdditions.size(),3);
    assertEquals(amClient.blacklistRemovals.size(),0);
    
    // Add nodes[1] and nodes[2] to removal list, 
    // Verify addition list remove these two nodes.
    List<String> nodeList12 = new ArrayList<String>();
    nodeList12.add(nodes[1]);
    nodeList12.add(nodes[2]);
    amClient.updateBlacklist(null, nodeList12);
    assertEquals(amClient.blacklistAdditions.size(),1);
    assertEquals(amClient.blacklistRemovals.size(),2);
    
    // Add nodes[1] again to addition list, 
    // Verify removal list will remove this node.
    List<String> nodeList1 = new ArrayList<String>();
    nodeList1.add(nodes[1]);
    amClient.updateBlacklist(nodeList1, null);
    assertEquals(amClient.blacklistAdditions.size(),2);
    assertEquals(amClient.blacklistRemovals.size(),1);
  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}