Java DAG.OperatorMeta方法代码示例

import com.datatorrent.api.DAG; //导入方法依赖的package包/类
@Test
public void test() throws Exception {
  ApexPipelineOptions options = PipelineOptionsFactory.create()
      .as(ApexPipelineOptions.class);
  EmbeddedCollector.RESULTS.clear();
  options.setApplicationName("ReadUnbound");
  options.setRunner(ApexRunner.class);
  Pipeline p = Pipeline.create(options);

  List<String> collection = Lists.newArrayList("1", "2", "3", "4", "5");
  CollectionSource<String> source = new CollectionSource<>(collection, StringUtf8Coder.of());
  p.apply(Read.from(source))
      .apply(ParDo.of(new EmbeddedCollector()));

  ApexRunnerResult result = (ApexRunnerResult) p.run();
  DAG dag = result.getApexDAG();
  DAG.OperatorMeta om = dag.getOperatorMeta("Read(CollectionSource)");
  Assert.assertNotNull(om);
  Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class);

  long timeout = System.currentTimeMillis() + 30000;
  while (System.currentTimeMillis() < timeout) {
    if (EmbeddedCollector.RESULTS.containsAll(collection)) {
      break;
    }
    LOG.info("Waiting for expected results.");
    Thread.sleep(1000);
  }
  Assert.assertEquals(Sets.newHashSet(collection), EmbeddedCollector.RESULTS);
}
 

import com.datatorrent.api.DAG; //导入方法依赖的package包/类
@Test
public void testReadBounded() throws Exception {
  ApexPipelineOptions options = PipelineOptionsFactory.create()
      .as(ApexPipelineOptions.class);
  EmbeddedCollector.RESULTS.clear();
  options.setApplicationName("ReadBounded");
  options.setRunner(ApexRunner.class);
  Pipeline p = Pipeline.create(options);

  Set<Long> expected = ContiguousSet.create(Range.closedOpen(0L, 10L), DiscreteDomain.longs());
  p.apply(GenerateSequence.from(0).to(10))
      .apply(ParDo.of(new EmbeddedCollector()));

  ApexRunnerResult result = (ApexRunnerResult) p.run();
  DAG dag = result.getApexDAG();
  String operatorName = "GenerateSequence/Read(BoundedCountingSource)";
  DAG.OperatorMeta om = dag.getOperatorMeta(operatorName);
  Assert.assertNotNull(om);
  Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class);

  long timeout = System.currentTimeMillis() + 30000;
  while (System.currentTimeMillis() < timeout) {
    if (EmbeddedCollector.RESULTS.containsAll(expected)) {
      break;
    }
    LOG.info("Waiting for expected results.");
    Thread.sleep(1000);
  }
  Assert.assertEquals(Sets.newHashSet(expected), EmbeddedCollector.RESULTS);
}
 

import com.datatorrent.api.DAG; //导入方法依赖的package包/类
@Test
public void test() throws Exception {
  ApexPipelineOptions options = PipelineOptionsFactory.create()
      .as(ApexPipelineOptions.class);
  options.setApplicationName("ParDoBound");
  options.setRunner(ApexRunner.class);

  Pipeline p = Pipeline.create(options);

  List<Integer> collection = Lists.newArrayList(1, 2, 3, 4, 5);
  List<Integer> expected = Lists.newArrayList(6, 7, 8, 9, 10);
  p.apply(Create.of(collection).withCoder(SerializableCoder.of(Integer.class)))
      .apply(ParDo.of(new Add(5)))
      .apply(ParDo.of(new EmbeddedCollector()));

  ApexRunnerResult result = (ApexRunnerResult) p.run();
  DAG dag = result.getApexDAG();

  DAG.OperatorMeta om = dag.getOperatorMeta("Create.Values");
  Assert.assertNotNull(om);
  Assert.assertEquals(om.getOperator().getClass(), ApexReadUnboundedInputOperator.class);

  om = dag.getOperatorMeta("ParDo(Add)/ParMultiDo(Add)");
  Assert.assertNotNull(om);
  Assert.assertEquals(om.getOperator().getClass(), ApexParDoOperator.class);

  long timeout = System.currentTimeMillis() + TIMEOUT_MILLIS;
  while (System.currentTimeMillis() < timeout) {
    if (EmbeddedCollector.RESULTS.containsAll(expected)) {
      break;
    }
    LOG.info("Waiting for expected results.");
    Thread.sleep(SLEEP_MILLIS);
  }
  Assert.assertEquals(Sets.newHashSet(expected), EmbeddedCollector.RESULTS);
}
 

import com.datatorrent.api.DAG; //导入方法依赖的package包/类
@Override
public void handle(DAGSetupEvent event)
{
  for (DAG.OperatorMeta ometa : dag.getAllOperatorsMeta()) {
    Operator o = ometa.getOperator();
    LogicalPlanConfiguration.setOperatorProperties(o, propertyMap);
  }
}
 

import com.datatorrent.api.DAG; //导入方法依赖的package包/类
/**
 * Build the command to launch the child VM in the container
 *
 * @param jvmID
 * @return
 */
public List<CharSequence> getChildVMCommand(String jvmID)
{

  List<CharSequence> vargs = new ArrayList<>(8);

  if (!StringUtils.isBlank(System.getenv(Environment.JAVA_HOME.key()))) {
    // node manager provides JAVA_HOME
    vargs.add(Environment.JAVA_HOME.$() + "/bin/java");
  } else {
    vargs.add("java");
  }

  String jvmOpts = dag.getAttributes().get(LogicalPlan.CONTAINER_JVM_OPTIONS);
  if (jvmOpts == null) {
    if (dag.isDebug()) {
      vargs.add(JAVA_REMOTE_DEBUG_OPTS);
    }
  } else {
    Map<String, String> params = new HashMap<>();
    params.put("applicationId", Integer.toString(container.getId().getApplicationAttemptId().getApplicationId().getId()));
    params.put("containerId", Integer.toString(container.getId().getId()));
    StrSubstitutor sub = new StrSubstitutor(params, "%(", ")");
    vargs.add(sub.replace(jvmOpts));
    if (dag.isDebug() && !jvmOpts.contains("-agentlib:jdwp=")) {
      vargs.add(JAVA_REMOTE_DEBUG_OPTS);
    }
  }

  List<DAG.OperatorMeta> operatorMetaList = Lists.newArrayList();
  int bufferServerMemory = 0;
  for (PTOperator operator : sca.getContainer().getOperators()) {
    bufferServerMemory += operator.getBufferServerMemory();
    operatorMetaList.add(operator.getOperatorMeta());
  }
  Context.ContainerOptConfigurator containerOptConfigurator = dag.getAttributes().get(LogicalPlan.CONTAINER_OPTS_CONFIGURATOR);
  jvmOpts = containerOptConfigurator.getJVMOptions(operatorMetaList);
  jvmOpts = parseJvmOpts(jvmOpts, ((long)bufferServerMemory) * MB_TO_B);
  LOG.info("Jvm opts {} for container {}",jvmOpts,container.getId());
  vargs.add(jvmOpts);

  Path childTmpDir = new Path(Environment.PWD.$(), YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR);
  vargs.add(String.format("-D%s=%s", StreamingContainer.PROP_APP_PATH, dag.assertAppPath()));
  vargs.add("-Djava.io.tmpdir=" + childTmpDir);
  vargs.add(String.format("-D%scid=%s", StreamingApplication.DT_PREFIX, jvmID));
  vargs.add("-Dhadoop.root.logger=" + (dag.isDebug() ? "DEBUG" : "INFO") + ",RFA");
  vargs.add("-Dhadoop.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR);
  StramClientUtils.addAttributeToArgs(LogicalPlan.APPLICATION_NAME, dag, vargs);
  StramClientUtils.addAttributeToArgs(LogicalPlan.LOGGER_APPENDER, dag, vargs);

  String loggersLevel = System.getProperty(StramUtils.DT_LOGGERS_LEVEL);
  if (loggersLevel != null) {
    vargs.add(String.format("-D%s=%s", StramUtils.DT_LOGGERS_LEVEL, loggersLevel));
  }
  // Add main class and its arguments
  vargs.add(StreamingContainer.class.getName());  // main of Child

  vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
  vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");

  // Final commmand
  StringBuilder mergedCommand = new StringBuilder(256);
  for (CharSequence str : vargs) {
    mergedCommand.append(str).append(" ");
  }
  List<CharSequence> vargsFinal = new ArrayList<>(1);
  vargsFinal.add(mergedCommand.toString());
  return vargsFinal;

}
 

import com.datatorrent.api.DAG; //导入方法依赖的package包/类
@Override
public String getJVMOptions(List<DAG.OperatorMeta> operatorMetaList)
{
  Set<String> genericProperties = null;
  long xmx = 0;
  long xms = 0;
  long xss = 0;
  List<Map<String, Object>> jvmOptsList = Lists.newArrayList();
  for (DAG.OperatorMeta operatorMeta : operatorMetaList) {
    Map<String, Object> operatorMap = parseJvmOpts(operatorMeta.getValue(Context.OperatorContext.JVM_OPTIONS), operatorMeta.getValue(Context.OperatorContext.MEMORY_MB));
    LOG.info("property map for operator {}", operatorMap);
    Set<String> operatorPropertySet = (Set<String>)operatorMap.get(GENERIC);
    if (genericProperties == null) {
      genericProperties = operatorPropertySet;
    } else {
      if (operatorPropertySet != null && !genericProperties.equals(operatorPropertySet)) {
        throw new AssertionError("Properties don't match: " + genericProperties + " " + operatorPropertySet);
      }
    }
    jvmOptsList.add(operatorMap);
  }
  for (Map<String, Object> map : jvmOptsList) {
    String value;
    if (map.containsKey(XMX)) {
      value = (String)map.get(XMX);
      xmx += getOptValue(value);
    }
    if (map.containsKey(XMS)) {
      value = (String)map.get(XMS);
      xms += getOptValue(value);
    }
    if (map.containsKey(XSS)) {
      value = (String)map.get(XSS);
      xss += getOptValue(value);
    }
  }
  StringBuilder builder = new StringBuilder(" ");
  builder.append(XMX).append(xmx);
  if (xms != 0) {
    builder.append(" ").append(XMS).append(xms);
  }
  if (xss != 0) {
    builder.append(" ").append(XSS).append(xss);
  }
  if (genericProperties != null) {
    for (String property : genericProperties) {
      builder.append(" ").append(property);
    }
  }
  return builder.toString();
}