Java TwillController.getResourceReport方法代码示例

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private boolean waitForDebugPort(TwillController controller, String runnable, int timeLimit)
  throws InterruptedException {
  long millis = 0;
  while (millis < 1000 * timeLimit) {
    ResourceReport report = controller.getResourceReport();
    if (report == null || report.getRunnableResources(runnable) == null) {
      continue;
    }
    for (TwillRunResources resources : report.getRunnableResources(runnable)) {
      if (resources.getDebugPort() != null) {
        return true;
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
    millis += 100;
  }
  return false;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private boolean waitForLogLevel(TwillController controller, String runnable, long timeout,
                                TimeUnit timeoutUnit, @Nullable LogEntry.Level expected) throws InterruptedException {

  Stopwatch stopwatch = new Stopwatch();
  stopwatch.start();
  do {
    ResourceReport report = controller.getResourceReport();
    if (report == null || report.getRunnableResources(runnable) == null) {
      continue;
    }
    for (TwillRunResources resources : report.getRunnableResources(runnable)) {
      LogEntry.Level level = resources.getLogLevels().get(Logger.ROOT_LOGGER_NAME);
      if (expected == level) {
         return true;
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
  } while (stopwatch.elapsedTime(timeoutUnit) < timeout);

  return false;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private void waitForContainers(TwillController controller, int count, long timeout, TimeUnit timeoutUnit)
  throws Exception {
  Stopwatch stopwatch = new Stopwatch();
  stopwatch.start();
  int yarnContainers = 0;
  int twillContainers = 0;
  do {
    if (controller.getResourceReport() != null) {
      yarnContainers =
        getApplicationResourceReport(controller.getResourceReport().getApplicationId()).getNumUsedContainers();
      twillContainers = getTwillContainersUsed(controller);
      if (yarnContainers == count && twillContainers == count) {
        return;
      }
    }
    TimeUnit.SECONDS.sleep(1);
  } while (stopwatch.elapsedTime(timeoutUnit) < timeout);

  throw new TimeoutException("Timeout reached while waiting for num containers to be " +  count +
                               ". Yarn containers = " + yarnContainers + ", Twill containers = " + twillContainers);
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private void waitForInstance(TwillController controller, String runnable, String yarnInstanceId,
                             long timeout, TimeUnit timeoutUnit) throws InterruptedException, TimeoutException {
  Stopwatch stopwatch = new Stopwatch();
  stopwatch.start();
  do {
    ResourceReport report = controller.getResourceReport();
    if (report != null && report.getRunnableResources(runnable) != null) {
      for (TwillRunResources resources : report.getRunnableResources(runnable)) {
        if (resources.getContainerId().endsWith(yarnInstanceId)) {
          return;
        }
      }
    }
    TimeUnit.SECONDS.sleep(1);
  } while (stopwatch.elapsedTime(timeoutUnit) < timeout);

  throw new TimeoutException("Timeout reached while waiting for runnable " +
                               runnable + " instance " + yarnInstanceId);
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
/**
 * Terminates all instances of the {@link PeriodicNotificationTwillApp} on the YARN cluster.
 */
public void stopApp() {
    LOG.info("Stopping any running instances...");

    int counter = 0;
    // It is possible that we have launched multiple instances of the app.  For now, stop them all, one at a time.
    for(final TwillController c : twillRunner.lookup(PeriodicNotificationTwillApp.APPLICATION_NAME)) {
        final ResourceReport report = c.getResourceReport();
        LOG.info("Attempting to stop {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId());
        Futures.getUnchecked(c.terminate());
        LOG.info("Stopped {} with YARN ApplicationId: {} and Twill RunId: {}", PeriodicNotificationTwillApp.APPLICATION_NAME, report.getApplicationId(), c.getRunId());
        counter++;
    }

    LOG.info("Stopped {} instance(s) of {}", counter, PeriodicNotificationTwillApp.APPLICATION_NAME);
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
/**
 * Blocks until a non-null Resource report is returned.
 * @param controller - The controller to interrogate.
 * @param timeout - The maximum time to poll {@controller}.  Use -1 for infinite polling.
 * @param timeoutUnits - The units of {@code timeout}.
 * @return The ResourceReport for the application.
 * @throws IllegalStateException If a timeout occurs before a ResourceReport is returned.
 */
private ResourceReport getResourceReport(final TwillController controller, final long timeout, final TimeUnit timeoutUnits) {
    Preconditions.checkArgument(timeout >= -1, "timeout cannot be less than -1");
    final long timeoutMillis = TimeUnit.MILLISECONDS.convert(timeout, timeoutUnits);
    final long sleepMillis = 1000; // how long to sleep between retrieval attempts.
    long totalElapsedMillis = 0;
    ResourceReport report = controller.getResourceReport();
    while (reportIsLoading(report)) {
        try {
            Thread.sleep(sleepMillis);
        } catch (final InterruptedException e) {
            throw new IllegalStateException(e);
        }
        totalElapsedMillis += sleepMillis;
        if ((timeout != -1) && (totalElapsedMillis >= timeoutMillis)) {
            final String errorMessage = "Timeout while waiting for the Twill Application to start on YARN.  Total elapsed time: " + TimeUnit.SECONDS.convert(totalElapsedMillis, TimeUnit.MILLISECONDS) + "s.";
            LOG.error(errorMessage);
            throw new IllegalStateException(errorMessage);
        }
        if ((totalElapsedMillis % 5000) == 0) {
            LOG.info("Waiting for the Twill Application to start on YARN... Total elapsed time: {}s.", TimeUnit.SECONDS.convert(totalElapsedMillis, TimeUnit.MILLISECONDS));
        }
        report = controller.getResourceReport();
    }
    return report;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
/**
 * Attempts to retrieves ResourceReport until maxWaitMs time is reached. Set maxWaitMs to -1 to
 * retry forever.
 */
private ResourceReport getResourceReport(TwillController controller, int maxWaitMs) {
  ResourceReport report = controller.getResourceReport();
  int elapsed = 0;
  while (report == null) {
    report = controller.getResourceReport();
    try {
      Thread.sleep(500);
    } catch (InterruptedException e) {
      throw new IllegalStateException(e);
    }
    elapsed += 500;
    if ((maxWaitMs != -1) && (elapsed > maxWaitMs)) {
      String msg = String.format("Exceeded max wait time to retrieve ResourceReport from Twill."
          + " Elapsed time = %s ms", elapsed);
      log.error(msg);
      throw new IllegalStateException(msg);
    }
    if ((elapsed % 10000) == 0) {
      log.info("Waiting for ResourceReport from Twill. Elapsed time = {} ms", elapsed);
    }
  }
  return report;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private ResourceReport getResourceReport(TwillController controller, long timeoutMillis) {
  ResourceReport report = controller.getResourceReport();
  Stopwatch stopwatch = new Stopwatch();
  while (report == null && stopwatch.elapsedMillis() < timeoutMillis) {
    Uninterruptibles.sleepUninterruptibly(200, TimeUnit.MILLISECONDS);
    report = controller.getResourceReport();
  }

  Assert.assertNotNull(report);
  return report;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private int getTwillContainersUsed(TwillController controller) {
  if (controller.getResourceReport() == null) {
    return 0;
  }

  int count = 1; // 1 for app master container
  ResourceReport resourceReport = controller.getResourceReport();
  for (Collection<TwillRunResources> resources : resourceReport.getResources().values()) {
    count += resources.size();
  }
  return count;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
private void waitForLogLevel(TwillController controller, String runnable, long timeout,
                             TimeUnit timeoutUnit, LogEntry.Level expected,
                             Map<String, LogEntry.Level> expectedArgs) throws InterruptedException {

  Stopwatch stopwatch = new Stopwatch();
  stopwatch.start();
  LogEntry.Level actual = null;
  Map<String, LogEntry.Level> actualArgs = null;
  boolean stopped = false;
  do {
    ResourceReport report = controller.getResourceReport();
    if (report == null || report.getRunnableResources(runnable) == null) {
      continue;
    }
    for (TwillRunResources resources : report.getRunnableResources(runnable)) {
      actual = resources.getLogLevels().get(Logger.ROOT_LOGGER_NAME);
      actualArgs = resources.getLogLevels();
      if (actual != null && actual.equals(expected)) {
        stopped = true;
        break;
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
  } while (!stopped && stopwatch.elapsedTime(timeoutUnit) < timeout);
  Assert.assertEquals(expected, actual);
  Assert.assertEquals(expectedArgs, actualArgs);
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
@Override
public ClusterEnriched getClusterInfo(final Cluster cluster) throws YarnProvisioningHandlingException {
  // get info about cluster
  // children should do the rest
  TwillController controller = getTwillControllerHelper(cluster);
  final ClusterEnriched clusterEnriched = new ClusterEnriched(cluster);
  if (controller != null) {
    ResourceReport report = controller.getResourceReport();
    if (report == null) {
      return clusterEnriched;
    }
    Collection<TwillRunResources> runResources = report.getRunnableResources(DacDaemonYarnApplication.YARN_RUNNABLE_NAME);

    Set<String> activeContainerIds = new HashSet<>();
    for(NodeEndpoint ep : executionNodeProvider.getNodes()){
      if(ep.hasProvisionId()){
        activeContainerIds.add(ep.getProvisionId());
      }
    }

    Containers containers = new Containers();
    clusterEnriched.setRunTimeInfo(containers);

    List<Container> runningList = new ArrayList<Container>();
    List<Container> disconnectedList = new ArrayList<Container>();
    clusterEnriched.getRunTimeInfo().setRunningList(runningList);
    clusterEnriched.getRunTimeInfo().setDisconnectedList(disconnectedList);

    for (TwillRunResources runResource : runResources) {
      Container container = new Container();
      container.setContainerId(runResource.getContainerId());
      container.setContainerPropertyList(new ArrayList<Property>());
      container.getContainerPropertyList().add(new Property("host", runResource.getHost()));
      container.getContainerPropertyList().add(new Property("memoryMB", "" + runResource.getMemoryMB()));
      container.getContainerPropertyList().add(new Property("virtualCoreCount", "" + runResource.getVirtualCores()));
      if(activeContainerIds.contains(runResource.getContainerId())) {
        runningList.add(container);
      } else {
        disconnectedList.add(container);
      }
    }
    int yarnDelta = (cluster.getClusterConfig().getClusterSpec().getContainerCount() - runResources.size());
    if (yarnDelta > 0) {

      // pending
      clusterEnriched.getRunTimeInfo().setPendingCount(yarnDelta);
    } else if (yarnDelta < 0) {
      // decomissioning kind of
      clusterEnriched.getRunTimeInfo().setDecommissioningCount(Math.abs(yarnDelta));
    }

    clusterEnriched.getRunTimeInfo().setProvisioningCount(disconnectedList.size());

  }
  return clusterEnriched;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
@Test
public void testMaxHeapSize() throws InterruptedException, TimeoutException, ExecutionException {
  TwillRunner runner = getTwillRunner();

  TwillController controller = runner.prepare(new MaxHeapApp())
    // Alter the AM container size and heap ratio
    .withConfiguration(ImmutableMap.of(Configs.Keys.YARN_AM_MEMORY_MB, "256",
                                       Configs.Keys.HEAP_RESERVED_MIN_RATIO, "0.65"))
    // Use a different heap ratio and reserved memory size for the runnable
    .withConfiguration("sleep",
                       ImmutableMap.of(Configs.Keys.HEAP_RESERVED_MIN_RATIO, "0.8",
                                       Configs.Keys.JAVA_RESERVED_MEMORY_MB, "1024"))
    .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
    .start();

  try {
    ServiceDiscovered discovered = controller.discoverService("sleep");
    Assert.assertTrue(waitForSize(discovered, 2, 120));

    // Verify the AM container size and heap size
    ResourceReport resourceReport = controller.getResourceReport();
    Assert.assertNotNull(resourceReport);
    Assert.assertEquals(256, resourceReport.getAppMasterResources().getMemoryMB());
    Assert.assertEquals(Resources.computeMaxHeapSize(256, Configs.Defaults.YARN_AM_RESERVED_MEMORY_MB, 0.65d),
                        resourceReport.getAppMasterResources().getMaxHeapMemoryMB());

    // Verify the runnable container heap size
    Collection<TwillRunResources> runnableResources = resourceReport.getRunnableResources("sleep");
    Assert.assertFalse(runnableResources.isEmpty());
    TwillRunResources resources = runnableResources.iterator().next();
    Assert.assertEquals(Resources.computeMaxHeapSize(resources.getMemoryMB(), 1024, 0.8d),
                        resources.getMaxHeapMemoryMB());

    // For the sleep2 runnable, we don't set any ratio and reserved memory.
    // The ratio should get default to 0.65 (app) and reserved memory to 200
    runnableResources = resourceReport.getRunnableResources("sleep2");
    Assert.assertFalse(runnableResources.isEmpty());
    resources = runnableResources.iterator().next();
    Assert.assertEquals(
      Resources.computeMaxHeapSize(resources.getMemoryMB(), Configs.Defaults.YARN_AM_RESERVED_MEMORY_MB, 0.65d),
      resources.getMaxHeapMemoryMB());
  } finally {
    controller.terminate().get(120, TimeUnit.SECONDS);
  }
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
/**
 *  Need helper method here to wait for getting resource report because {@link TwillController#getResourceReport()}
 *  could return null if the application has not fully started.
 *
 *  This method helps validate restart scenario.
 *
 *  To avoid long sleep if instanceIdToContainerId is passed, then compare the container ids to ones before.
 *  Otherwise just return the valid resource report.
 */
@Nullable
private ResourceReport waitForAfterRestartResourceReport(TwillController controller, String runnable, long timeout,
                                                         TimeUnit timeoutUnit, int numOfResources,
                                                         @Nullable Map<Integer, String> instanceIdToContainerId) {
  Stopwatch stopwatch = new Stopwatch();
  stopwatch.start();
  do {
    ResourceReport report = controller.getResourceReport();
    if (report == null || report.getRunnableResources(runnable) == null) {
      Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
    } else if (report.getRunnableResources(runnable) == null ||
        report.getRunnableResources(runnable).size() != numOfResources) {
      Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
    } else {
      if (instanceIdToContainerId == null) {
        LOG.info("Return resource report without comparing container ids.");
        return report;
      }
      Collection<TwillRunResources> runResources = report.getRunnableResources(runnable);
      boolean isSameContainer = false;
      for (TwillRunResources twillRunResources : runResources) {
        int instanceId = twillRunResources.getInstanceId();
        if (twillRunResources.getContainerId().equals(instanceIdToContainerId.get(instanceId))) {
          // found same container id lets wait again.
          LOG.warn("Found an instance id {} with same container id {} for restart all, let's wait for a while.",
                   instanceId, twillRunResources.getContainerId());
          isSameContainer = true;
          break;
        }
      }
      if (!isSameContainer) {
        LOG.info("Get set of different container ids for restart.");
        return report;
      }
      Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
    }
  } while (stopwatch.elapsedTime(timeoutUnit) < timeout);

  LOG.error("Unable to get different container ids for restart.");
  return null;
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
@Test(timeout = 120000L)
public void testLocationCache() throws Exception {
  TwillRunner twillRunner = TWILL_TESTER.getTwillRunner();

  // Start the runnable
  TwillController controller = twillRunner.prepare(new BlockingTwillRunnable())
    .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
    .start();

  // Wait until the runnable is runnable
  String runnableName = BlockingTwillRunnable.class.getSimpleName();
  ResourceReport resourceReport = controller.getResourceReport();
  while (resourceReport == null || resourceReport.getRunnableResources(runnableName).isEmpty()) {
    TimeUnit.SECONDS.sleep(1);
    resourceReport = controller.getResourceReport();
  }

  long startTime = System.currentTimeMillis();

  // Inspect the cache directory, there should be a directory, which is the current session
  // inside that directory, there should be three files, launcher.jar, twill.jar and an application jar
  LocationFactory locationFactory = TWILL_TESTER.createLocationFactory();
  Location cacheBase = locationFactory.create(".cache");

  List<Location> cacheDirs = cacheBase.list();
  Assert.assertEquals(1, cacheDirs.size());

  Location currentSessionCache = cacheDirs.get(0);
  Assert.assertEquals(3, currentSessionCache.list().size());

  // Force a cleanup of cache. The first call is to collect the locations to be cleanup.
  // The second call is the actual cleanup.
  ((YarnTwillRunnerService) twillRunner).forceLocationCacheCleanup(startTime);
  ((YarnTwillRunnerService) twillRunner).forceLocationCacheCleanup(startTime +
                                                                     Configs.Defaults.LOCATION_CACHE_EXPIRY_MS);

  // Since the app is still runnable, no files in the cache should get removed.
  Assert.assertEquals(3, currentSessionCache.list().size());

  // Stop the app
  controller.terminate().get();

  // Force a cleanup of cache. The first call is to collect the locations to be cleanup.
  // The second call is the actual cleanup.
  ((YarnTwillRunnerService) twillRunner).forceLocationCacheCleanup(startTime);
  ((YarnTwillRunnerService) twillRunner).forceLocationCacheCleanup(startTime +
                                                                     Configs.Defaults.LOCATION_CACHE_EXPIRY_MS);

  // Since the app is stopped, there should only be two files, the launcher.jar and twill.jar, as they
  // will never get removed for the current session.
  Set<Location> cachedLocations = new HashSet<>(currentSessionCache.list());
  Assert.assertEquals(2, cachedLocations.size());
  Assert.assertTrue(cachedLocations.contains(currentSessionCache.append(Constants.Files.LAUNCHER_JAR)));
  Assert.assertTrue(cachedLocations.contains(currentSessionCache.append(Constants.Files.TWILL_JAR)));

  // Start another YarnTwillRunnerService
  TwillRunnerService newTwillRunner = TWILL_TESTER.createTwillRunnerService();
  newTwillRunner.start();

  // Force a cleanup using the antique expiry. The list of locations that need to be cleanup was already
  // collected when the new twill runner was started
  ((YarnTwillRunnerService) newTwillRunner)
    .forceLocationCacheCleanup(System.currentTimeMillis() + Configs.Defaults.LOCATION_CACHE_ANTIQUE_EXPIRY_MS);

  // Now there shouldn't be any file under the current session cache directory
  Assert.assertTrue(currentSessionCache.list().isEmpty());
}
 

import org.apache.twill.api.TwillController; //导入方法依赖的package包/类
@Test
public void testResourceReportWithFailingContainers() throws InterruptedException, IOException,
  TimeoutException, ExecutionException {
  TwillRunner runner = YarnTestUtils.getTwillRunner();

  ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
    .setVirtualCores(1)
    .setMemory(256, ResourceSpecification.SizeUnit.MEGA)
    .setInstances(2)
    .build();
  TwillController controller = runner.prepare(new BuggyServer(), resourceSpec)
    .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
    .withApplicationArguments("echo")
    .withArguments("BuggyServer", "echo2")
    .start();

  final CountDownLatch running = new CountDownLatch(1);
  controller.addListener(new ServiceListenerAdapter() {
    @Override
    public void running() {
      running.countDown();
    }
  }, Threads.SAME_THREAD_EXECUTOR);

  Assert.assertTrue(running.await(120, TimeUnit.SECONDS));

  Iterable<Discoverable> echoServices = controller.discoverService("echo");
  Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 2, 120));
  // check that we have 2 runnables.
  ResourceReport report = controller.getResourceReport();
  Assert.assertEquals(2, report.getRunnableResources("BuggyServer").size());

  // cause a divide by 0 in one server
  Discoverable discoverable = echoServices.iterator().next();
  Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
                             discoverable.getSocketAddress().getPort());
  try {
    PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
    writer.println("0");
  } finally {
    socket.close();
  }

  // takes some time for app master to find out the container completed...
  int count = 0;
  while (count < 100) {
    report = controller.getResourceReport();
    // check that we have 1 runnable, not 2.
    if (report.getRunnableResources("BuggyServer").size() == 1) {
      break;
    }
    LOG.info("Wait for BuggyServer to have 1 instance left. Trial {}.", count);
    count++;
    TimeUnit.SECONDS.sleep(1);
  }
  Assert.assertTrue("Still has 2 contains running after 100 seconds", count < 100);

  controller.stop().get(100, TimeUnit.SECONDS);
  // Sleep a bit before exiting.
  TimeUnit.SECONDS.sleep(2);
}