Java Launcher类代码示例

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Override
protected Launcher<?> getApexLauncher() {
  return new Launcher<AppHandle>() {
    @Override
    public AppHandle launchApp(StreamingApplication application,
        Configuration configuration, AttributeMap launchParameters)
        throws org.apache.apex.api.Launcher.LauncherException {
      EmbeddedAppLauncher<?> embeddedLauncher = Launcher.getLauncher(LaunchMode.EMBEDDED);
      DAG dag = embeddedLauncher.getDAG();
      application.populateDAG(dag, new Configuration(false));
      String appName = dag.getValue(DAGContext.APPLICATION_NAME);
      Assert.assertEquals("DummyApp", appName);
      return new AppHandle() {
        @Override
        public boolean isFinished() {
          return true;
        }
        @Override
        public void shutdown(org.apache.apex.api.Launcher.ShutdownMode arg0) {
        }
      };
    }
  };
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception
{
  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
  SimpleTransformApplication simpleTransformApplication = new SimpleTransformApplication();
  simpleTransformApplication.outputFn = outputFn;
  Launcher.AppHandle appHandle = launcher.launchApp(simpleTransformApplication, conf, launchAttributes);
  int sleepTimeCounterForLoopExit = 0;
  int sleepTimePerIteration = 500;
  // wait until expected result count or timeout
  while (results.size() < simpleTransformApplication.pojoDataGenerator.getMaxTuples()) {
    sleepTimeCounterForLoopExit += sleepTimePerIteration;
    if (sleepTimeCounterForLoopExit > 30000) {
      break;
    }
    Thread.sleep(sleepTimePerIteration);
  }
  appHandle.shutdown(Launcher.ShutdownMode.KILL);
  assertEquals(results.size(), simpleTransformApplication.pojoDataGenerator.getMaxTuples());
  assertTransformationsGenerated(simpleTransformApplication);
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception
{
  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(Launcher.LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
  EnricherAppWithJSONFile enricherAppWithJSONFile = new EnricherAppWithJSONFile();
  enricherAppWithJSONFile.outputFn = outputFn;
  Launcher.AppHandle appHandle = launcher.launchApp(enricherAppWithJSONFile, conf, launchAttributes);
  int sleepTimeCounterForLoopExit = 0;
  int sleepTimePerIteration = 500;
  // wait until expected result count or timeout
  while (results.size() < enricherAppWithJSONFile.getDataGenerator().getLimit()) {
    sleepTimeCounterForLoopExit += sleepTimePerIteration;
    if (sleepTimeCounterForLoopExit > 30000) {
      break;
    }
    Thread.sleep(sleepTimePerIteration);
  }
  appHandle.shutdown(ShutdownMode.KILL);
  assertTrue(results.size() >= enricherAppWithJSONFile.getDataGenerator().getLimit());
  assertEnrichedOutput();
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception {
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  //launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, false);
  //launchAttributes.put(EmbeddedAppLauncher.HEARTBEAT_MONITORING, false);
  Configuration conf = new Configuration(false);
  conf.addResource(new Path(TEMPLATE_PROPERTIES_PATH));
  conf.addResource(new Path(PROPERTIES_PATH));
  //conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.setBooleanIfUnset(TwitterStatsApp.IS_TWITTER_SAMPLE_INPUT, defaultTwitterInput);
  conf.setBooleanIfUnset(TwitterStatsApp.IS_WEBSOCKET_OUTPUT, defaultWebsocketOutput);

  StreamingApplication app = /*new TestApplication();*/ new TwitterStatsApp();
  AppHandle appHandle = launcher.launchApp(app, conf, launchAttributes);
  long timeoutMillis = System.currentTimeMillis() + Duration.standardMinutes(15).getMillis();
  while (!appHandle.isFinished() && System.currentTimeMillis() < timeoutMillis) {
    Thread.sleep(500);
  }
  appHandle.shutdown(ShutdownMode.KILL);
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
public AppHandle launchApp(StreamingApplication app, Properties configProperties)
    throws IOException {

  List<File> jarsToShip = getYarnDeployDependencies();
  StringBuilder classpath = new StringBuilder();
  for (File path : jarsToShip) {
    if (path.isDirectory()) {
      File tmpJar = File.createTempFile("beam-runners-apex-", ".jar");
      createJar(path, tmpJar);
      tmpJar.deleteOnExit();
      path = tmpJar;
    }
    if (classpath.length() != 0) {
      classpath.append(':');
    }
    classpath.append(path.getAbsolutePath());
  }

  EmbeddedAppLauncher<?> embeddedLauncher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  DAG dag = embeddedLauncher.getDAG();
  app.populateDAG(dag, new Configuration(false));

  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(YarnAppLauncher.LIB_JARS, classpath.toString().replace(':', ','));
  LaunchParams lp = new LaunchParams(dag, launchAttributes, configProperties);
  lp.cmd = "hadoop " + ApexYarnLauncher.class.getName();
  HashMap<String, String> env = new HashMap<>();
  env.put("HADOOP_USER_CLASSPATH_FIRST", "1");
  env.put("HADOOP_CLASSPATH", classpath.toString());
  lp.env = env;
  return launchApp(lp);
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception {
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
  //launchAttributes.put(EmbeddedAppLauncher.HEARTBEAT_MONITORING, false);
  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.addResource(this.getClass().getResourceAsStream("/wordcounttest-properties.xml"));

  File resultFile = new File("./target/wordcountresult_4.0");
  if (resultFile.exists() && !resultFile.delete()) {
    throw new AssertionError("Failed to delete " + resultFile);
  }
  AppHandle appHandle = launcher.launchApp(new Application(), conf, launchAttributes);
  long timeoutMillis = System.currentTimeMillis() + 30000;
  while (!appHandle.isFinished() && System.currentTimeMillis() < timeoutMillis) {
    Thread.sleep(500);
    if (resultFile.exists() && resultFile.length() > 0) {
      break;
    }
  }
  appHandle.shutdown(ShutdownMode.KILL);
  Assert.assertTrue(resultFile.exists() && resultFile.length() > 0);
  String result = FileUtils.readFileToString(resultFile);
  Assert.assertTrue(result, result.contains("MyFirstApplication=5"));
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception {
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
  //launchAttributes.put(EmbeddedAppLauncher.HEARTBEAT_MONITORING, false);
  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.addResource(this.getClass().getResourceAsStream("/wordcounttest-properties.xml"));

  File resultFile = new File("./target/wordcountresult_4.5");
  if (resultFile.exists() && !resultFile.delete()) {
    throw new AssertionError("Failed to delete " + resultFile);
  }
  AppHandle appHandle = launcher.launchApp(new Application(), conf, launchAttributes);
  long timeoutMillis = System.currentTimeMillis() + 10000;
  while (!appHandle.isFinished() && System.currentTimeMillis() < timeoutMillis) {
    Thread.sleep(500);
    if (resultFile.exists() && resultFile.length() > 0) {
      break;
    }
  }
  appHandle.shutdown(ShutdownMode.KILL);
  Assert.assertTrue(resultFile.exists() && resultFile.length() > 0);
  String result = FileUtils.readFileToString(resultFile);
  Assert.assertTrue(result, result.contains("="));
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testWithHighLevelAPI() throws Exception
{
  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true);
  Configuration conf = new Configuration(false);
  AppHandle appHandle = launcher.launchApp(new ApplicationWithHighLevelAPI(), conf, launchAttributes);
  long timeoutMillis = System.currentTimeMillis() + 10000;
  while (!appHandle.isFinished() && System.currentTimeMillis() < timeoutMillis) {
    Thread.sleep(500);
  }
  appHandle.shutdown(ShutdownMode.KILL);
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception {

  String outputFilePrefix = "target/wordcountresult.txt";
  File outFile1 = new File(outputFilePrefix + "-00000-of-00002");
  File outFile2 = new File(outputFilePrefix + "-00001-of-00002");
  Assert.assertTrue(!outFile1.exists() || outFile1.delete());
  Assert.assertTrue(!outFile2.exists() || outFile2.delete());

  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();

  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));

  PipelineOptionsFactory.register(WordCountOptions.class);
  WordCountOptions options = TestPipeline.testingPipelineOptions().as(WordCountOptions.class);
  options.setRunner(TestApexRunner.class);
  options.setInputFile(new File("./pom.xml").getAbsolutePath());
  options.setOutput(outputFilePrefix);
  // convert the options to command line string and pass it through the conf
  conf.set(Application.KEY_PIPELINE_OPTIONS, StringUtils.join(TestPipeline.convertToArgs(options), " "));
  AppHandle appHandle = launcher.launchApp(new Application(), conf, launchAttributes);

  Assert.assertTrue(appHandle.isFinished());
  Assert.assertTrue("result files exist", outFile1.exists() && outFile2.exists());
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
protected Launcher<?> getApexLauncher() {
  return Launcher.getLauncher(LaunchMode.YARN);
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception
{
  KafkaUnit ku = kafkaUnitRule.getKafkaUnit();
  String topicName = "testTopic";
  // topic creation is async and the producer may also auto-create it
  ku.createTopic(topicName, 1);

  // produce test data
  String[] words = "count the words from kafka and store them in the db".split("\\s+");
  for (String word : words) {
    ku.sendMessages(new KeyedMessage<String, String>(topicName, word));
  }

  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.set("apex.operator.kafkaInput.prop.topics", topicName);
  conf.set("apex.operator.kafkaInput.prop.clusters", "localhost:" + brokerPort);
  conf.set("apex.operator.kafkaInput.prop.maxTuplesPerWindow", "1"); // consume one word per window
  conf.set("apex.operator.kafkaInput.prop.initialOffset", "EARLIEST");
  conf.set("apex.operator.store.prop.store.databaseDriver", DB_DRIVER);
  conf.set("apex.operator.store.prop.store.databaseUrl", DB_URL);

  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true); // terminate after results are available
  AppHandle appHandle = launcher.launchApp(new ExactlyOnceJdbcOutputApp(), conf, launchAttributes);
  HashSet<String> wordsSet = Sets.newHashSet(words);
  Connection con = DriverManager.getConnection(DB_URL);
  Statement stmt = con.createStatement();
  int rowCount = 0;
  long timeout = System.currentTimeMillis() + 30000; // 30s timeout
  while (rowCount < wordsSet.size() && timeout > System.currentTimeMillis()) {
    Thread.sleep(500);
    String countQuery = "SELECT count(*) from " + TABLE_NAME;
    ResultSet resultSet = stmt.executeQuery(countQuery);
    resultSet.next();
    rowCount = resultSet.getInt(1);
    resultSet.close();
    LOG.info("current row count in {} is {}", TABLE_NAME, rowCount);
  }
  Assert.assertEquals("number of words", wordsSet.size(), rowCount);
  appHandle.shutdown(ShutdownMode.KILL);
}
 

import org.apache.apex.api.Launcher; //导入依赖的package包/类
@Test
public void testApplication() throws Exception
{
  File targetDir = new File(TARGET_DIR);
  FileUtils.deleteDirectory(targetDir);
  FileUtils.forceMkdir(targetDir);

  KafkaUnit ku = kafkaUnitRule.getKafkaUnit();
  String topicName = "testTopic";
  // topic creation is async and the producer may also auto-create it
  ku.createTopic(topicName, 1);

  // produce test data
  String[] words = "count count the words from kafka and store them in a file".split("\\s+");
  for (String word : words) {
    ku.sendMessages(new KeyedMessage<String, String>(topicName, word));
  }

  Configuration conf = new Configuration(false);
  conf.addResource(this.getClass().getResourceAsStream("/META-INF/properties.xml"));
  conf.set("apex.operator.kafkaInput.prop.topics", topicName);
  conf.set("apex.operator.kafkaInput.prop.clusters", "localhost:" + brokerPort);
  conf.set("apex.operator.kafkaInput.prop.maxTuplesPerWindow", "2"); // consume one word per window
  conf.set("apex.operator.kafkaInput.prop.initialOffset", "EARLIEST");
  conf.set("apex.operator.fileWriter.prop.filePath", TARGET_DIR);

  EmbeddedAppLauncher<?> launcher = Launcher.getLauncher(LaunchMode.EMBEDDED);
  Attribute.AttributeMap launchAttributes = new Attribute.AttributeMap.DefaultAttributeMap();
  launchAttributes.put(EmbeddedAppLauncher.RUN_ASYNC, true); // terminate after results are available
  AppHandle appHandle = launcher.launchApp(new ExactlyOnceFileOutputApp(), conf, launchAttributes);

  long timeout = System.currentTimeMillis() + 60000; // 60s timeout

  File outputFile = new File(TARGET_DIR, ExactlyOnceFileOutputApp.FileWriter.FILE_NAME_PREFIX);
  while (!outputFile.exists() && timeout > System.currentTimeMillis()) {
    Thread.sleep(1000);
    LOG.debug("Waiting for {}", outputFile);
  }

  Assert.assertTrue("output file exists " + ExactlyOnceFileOutputApp.FileWriter.FILE_NAME_PREFIX, outputFile.exists() &&
      outputFile.isFile());

  String result = FileUtils.readFileToString(outputFile);
  Assert.assertTrue(result.contains("count=2"));

  appHandle.shutdown(ShutdownMode.KILL);
}