Java YarnTwillRunnerService类代码示例

import org.apache.twill.yarn.YarnTwillRunnerService; //导入依赖的package包/类
/**
 *
 * @param yarnZookeepers - The zookeeper connect string used by the Hadoop YARN cluster.
 * @param configFile - The config file used by {@link PeriodicNotificationTwillApp}.  Typically notification.properties.
 */
public PeriodicNotificationTwillRunner(final String yarnZookeepers, final File configFile) {
    Preconditions.checkArgument(configFile.exists(), "Config File must exist");
    Objects.requireNonNull(yarnZookeepers, "YARN Zookeepers must not be null.");
    this.configFile = configFile;
    yarnConfiguration = new YarnConfiguration();
    twillRunner = new YarnTwillRunnerService(yarnConfiguration, yarnZookeepers);
    twillRunner.start();

    // sleep to give the YarnTwillRunnerService time to retrieve state from zookeeper
    try {
        Thread.sleep(1000);
    } catch (final InterruptedException e) {
        throw new IllegalStateException(e);
    }
}
 

import org.apache.twill.yarn.YarnTwillRunnerService; //导入依赖的package包/类
private synchronized TwillRunnerService getTwillRunner(FluoConfiguration config) {
  if (!twillRunners.containsKey(config.getApplicationName())) {
    YarnConfiguration yarnConfig = new YarnConfiguration();
    yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/core-site.xml"));
    yarnConfig.addResource(new Path(hadoopPrefix + "/etc/hadoop/yarn-site.xml"));

    TwillRunnerService twillRunner =
        new YarnTwillRunnerService(yarnConfig, config.getAppZookeepers() + ZookeeperPath.TWILL);
    twillRunner.start();

    twillRunners.put(config.getApplicationName(), twillRunner);

    // sleep to give twill time to retrieve state from zookeeper
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      throw new IllegalStateException(e);
    }
  }
  return twillRunners.get(config.getApplicationName());
}
 

import org.apache.twill.yarn.YarnTwillRunnerService; //导入依赖的package包/类
public TwillRunnerService startTwillRunner(YarnConfiguration yarnConfiguration) {
  String zkStr = dremioConfig.getString(DremioConfig.ZOOKEEPER_QUORUM);
  String clusterId = yarnConfiguration.get(YARN_CLUSTER_ID);
  Preconditions.checkNotNull(clusterId, "Cluster ID can not be null");
  TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
  TwillRunnerService previousOne = twillRunners.putIfAbsent(new ClusterId(clusterId), twillRunner);
  if (previousOne == null) {
    // start one we are planning to add - if it is already in collection it should be started
    twillRunner.start();
    return twillRunner;
  }
  return previousOne;
}
 

import org.apache.twill.yarn.YarnTwillRunnerService; //导入依赖的package包/类
public static void main(String[] args) {
  if (args.length < 1) {
    System.err.println("Arguments format: <host:port of zookeeper server>");
    System.exit(1);
  }

  String zkStr = args[0];
  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
  twillRunner.start();

  String yarnClasspath =
    yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                          Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
  List<String> applicationClassPaths = Lists.newArrayList();
  Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
  final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
      .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
      .withApplicationClassPaths(applicationClassPaths)
      .withBundlerClassAcceptor(new HadoopClassExcluder())
      .start();

  Runtime.getRuntime().addShutdownHook(new Thread() {
    @Override
    public void run() {
      try {
        Futures.getUnchecked(controller.terminate());
      } finally {
        twillRunner.stop();
      }
    }
  });

  try {
    controller.awaitTerminated();
  } catch (ExecutionException e) {
    e.printStackTrace();
  }
}
 

import org.apache.twill.yarn.YarnTwillRunnerService; //导入依赖的package包/类
public static void main(String[] args) {
  if (args.length < 3) {
    System.err.println("Arguments format: <host:port of zookeeper server>"
                         + " <bundle jar path> <main class name> <extra args>");
    System.exit(1);
  }

  String zkStr = args[0];
  BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(
          args[1], "/lib", args[2], Arrays.copyOfRange(args, 3, args.length));

  File jarFile = new File(arguments.getJarFileName());
  Preconditions.checkState(jarFile.exists());
  Preconditions.checkState(jarFile.canRead());

  final TwillRunnerService twillRunner = new YarnTwillRunnerService(new YarnConfiguration(), zkStr);
  twillRunner.start();


  final TwillController controller = twillRunner.prepare(
    new ExampleBundledJarApp(jarFile.getName(), jarFile.toURI()))
    .withArguments("BundledJarRunnable", arguments.toArray())
    .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
    .start();

  Runtime.getRuntime().addShutdownHook(new Thread() {
    @Override
    public void run() {
      try {
        Futures.getUnchecked(controller.terminate());
      } finally {
        twillRunner.stop();
      }
    }
  });

  try {
    controller.awaitTerminated();
  } catch (ExecutionException e) {
    LOG.error("Error", e);
  }
}