Java Create.Values方法代码示例

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
/**
 * Traverse the specified {@link PTransform}, collecting {@link DisplayData} registered on the
 * inner primitive {@link PTransform PTransforms}.
 *
 * @param root The root {@link PTransform} to traverse
 * @param inputCoder The coder to set for the {@link PTransform} input, or null to infer the
 *                   default coder.
 *
 * @return the set of {@link DisplayData} for primitive {@link PTransform PTransforms}.
 */
public <InputT> Set<DisplayData> displayDataForPrimitiveTransforms(
    final PTransform<? super PCollection<InputT>, ? extends POutput> root,
    Coder<InputT> inputCoder) {

  Create.Values<InputT> input;
  if (inputCoder != null) {
    input = Create.empty(inputCoder);
  } else {
    // These types don't actually work, but the pipeline will never be run
    input = (Create.Values<InputT>) Create.empty(VoidCoder.of());
  }

  Pipeline pipeline = Pipeline.create(options);
  pipeline
      .apply("Input", input)
      .apply("Transform", root);

  return displayDataForPipeline(pipeline, root);
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Test
public void producingOwnOutputWithCompositeFails() {
  final PCollection<Long> comp =
      PCollection.createPrimitiveOutputInternal(
          pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
  PTransform<PBegin, PCollection<Long>> root =
      new PTransform<PBegin, PCollection<Long>>() {
        @Override
        public PCollection<Long> expand(PBegin input) {
          return comp;
        }
      };
  hierarchy.pushNode("Composite", PBegin.in(pipeline), root);

  Create.Values<Integer> create = Create.of(1);
  hierarchy.pushNode("Create", PBegin.in(pipeline), create);
  hierarchy.setOutput(pipeline.apply(create));
  hierarchy.popNode();

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("contains a primitive POutput produced by it");
  thrown.expectMessage("primitive transforms are permitted to produce");
  thrown.expectMessage("Composite");
  hierarchy.setOutput(comp);
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
/**
 * Tests that trying to register a transform which has unregistered children throws.
 */
@Test
public void registerTransformWithUnregisteredChildren() throws IOException {
  Create.Values<Long> create = Create.of(1L, 2L, 3L);
  GenerateSequence createChild = GenerateSequence.from(0);

  PCollection<Long> pt = pipeline.apply(create);
  String userName = "my_transform";
  String childUserName = "my_transform/my_nesting";
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.<PBegin, PCollection<Long>, Create.Values<Long>>of(
          userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
  AppliedPTransform<?, ?, ?> childTransform =
      AppliedPTransform.<PBegin, PCollection<Long>, GenerateSequence>of(
          childUserName, pipeline.begin().expand(), pt.expand(), createChild, pipeline);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(childTransform.toString());
  components.registerPTransform(
      transform, Collections.<AppliedPTransform<?, ?, ?>>singletonList(childTransform));
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
private static <T> TransformEvaluator<Create.Values<T>> create() {
  return new TransformEvaluator<Create.Values<T>>() {
    @Override
    public void evaluate(Create.Values<T> transform, EvaluationContext context) {
      Iterable<T> elems = transform.getElements();
      // Use a coder to convert the objects in the PCollection to byte arrays, so they
      // can be transferred over the network.
      Coder<T> coder = context.getOutput(transform).getCoder();
      context.putBoundedDatasetFromValues(transform, elems, coder);
    }

    @Override
    public String toNativeString() {
      return "sparkContext.parallelize(Arrays.asList(...))";
    }
  };
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Test
public void testConfigProperties() throws Exception {

  String operName = "testProperties";
  ApexPipelineOptions options = PipelineOptionsFactory.create()
      .as(ApexPipelineOptions.class);

  // default configuration from class path
  Pipeline p = Pipeline.create();
  Create.Values<Void> empty = Create.empty(VoidCoder.of());
  p.apply(operName, empty);

  DAG dag = TestApexRunner.translate(p, options);
  OperatorMeta t1Meta = dag.getOperatorMeta(operName);
  Assert.assertNotNull(t1Meta);
  Assert.assertEquals(new Integer(32), t1Meta.getValue(OperatorContext.MEMORY_MB));

  File tmp = File.createTempFile("beam-runners-apex-", ".properties");
  tmp.deleteOnExit();
  Properties props = new Properties();
  props.setProperty("apex.operator." + operName + ".attr.MEMORY_MB", "64");
  try (FileOutputStream fos = new FileOutputStream(tmp)) {
    props.store(fos, "");
  }
  options.setConfigFile(tmp.getAbsolutePath());
  dag = TestApexRunner.translate(p, options);
  tmp.delete();

  t1Meta = dag.getOperatorMeta(operName);
  Assert.assertNotNull(t1Meta);
  Assert.assertEquals(new Integer(64), t1Meta.getValue(OperatorContext.MEMORY_MB));

}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Test
public void registerTransformNoChildren() throws IOException {
  Create.Values<Integer> create = Create.of(1, 2, 3);
  PCollection<Integer> pt = pipeline.apply(create);
  String userName = "my_transform/my_nesting";
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.<PBegin, PCollection<Integer>, Create.Values<Integer>>of(
          userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
  String componentName =
      components.registerPTransform(
          transform, Collections.<AppliedPTransform<?, ?, ?>>emptyList());
  assertThat(componentName, equalTo(userName));
  assertThat(components.getExistingPTransformId(transform), equalTo(componentName));
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Test
public void registerTransformEmptyFullName() throws IOException {
  Create.Values<Integer> create = Create.of(1, 2, 3);
  PCollection<Integer> pt = pipeline.apply(create);
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.<PBegin, PCollection<Integer>, Create.Values<Integer>>of(
          "", pipeline.begin().expand(), pt.expand(), create, pipeline);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(transform.toString());
  components.getExistingPTransformId(transform);
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Test
public void registerTransformNullComponents() throws IOException {
  Create.Values<Integer> create = Create.of(1, 2, 3);
  PCollection<Integer> pt = pipeline.apply(create);
  String userName = "my_transform/my_nesting";
  AppliedPTransform<?, ?, ?> transform =
      AppliedPTransform.<PBegin, PCollection<Integer>, Create.Values<Integer>>of(
          userName, pipeline.begin().expand(), pt.expand(), create, pipeline);
  thrown.expect(NullPointerException.class);
  thrown.expectMessage("child nodes may not be null");
  components.registerPTransform(transform, null);
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
    AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) {
  Create.Values<T> original = transform.getTransform();
  PCollection<T> output =
      (PCollection) Iterables.getOnlyElement(transform.getOutputs().values());
  return PTransformReplacement.of(
      transform.getPipeline().begin(),
      new StreamingFnApiCreate<>(original, output));
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
    AppliedPTransform<PBegin, PCollection<T>, Create.Values<T>> transform) {
  return PTransformReplacement.of(
      transform.getPipeline().begin(), new EmptyFlatten<T>());
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
@Test
public void replaceSucceeds() {
  PTransform<?, ?> enclosingPT =
      new PTransform<PInput, POutput>() {
        @Override
        public POutput expand(PInput input) {
          return PDone.in(input.getPipeline());
        }
      };

  TransformHierarchy.Node enclosing =
      hierarchy.pushNode("Enclosing", PBegin.in(pipeline), enclosingPT);

  Create.Values<Long> originalTransform = Create.of(1L);
  TransformHierarchy.Node original =
      hierarchy.pushNode("Create", PBegin.in(pipeline), originalTransform);
  assertThat(hierarchy.getCurrent(), equalTo(original));
  PCollection<Long> originalOutput = pipeline.apply(originalTransform);
  hierarchy.setOutput(originalOutput);
  hierarchy.popNode();
  assertThat(original.finishedSpecifying, is(true));
  hierarchy.setOutput(PDone.in(pipeline));
  hierarchy.popNode();

  assertThat(hierarchy.getCurrent(), not(equalTo(enclosing)));
  Read.Bounded<Long> replacementTransform = Read.from(CountingSource.upTo(1L));
  PCollection<Long> replacementOutput = pipeline.apply(replacementTransform);
  Node replacement = hierarchy.replaceNode(original, PBegin.in(pipeline), replacementTransform);
  assertThat(hierarchy.getCurrent(), equalTo(replacement));
  hierarchy.setOutput(replacementOutput);

  TaggedPValue taggedReplacement = TaggedPValue.ofExpandedValue(replacementOutput);
  Map<PValue, ReplacementOutput> replacementOutputs =
      Collections.<PValue, ReplacementOutput>singletonMap(
          replacementOutput,
          ReplacementOutput.of(
              TaggedPValue.ofExpandedValue(originalOutput),
              taggedReplacement));
  hierarchy.replaceOutputs(replacementOutputs);

  assertThat(replacement.getInputs(), equalTo(original.getInputs()));
  assertThat(replacement.getEnclosingNode(), equalTo(original.getEnclosingNode()));
  assertThat(replacement.getEnclosingNode(), equalTo(enclosing));
  assertThat(
      replacement.getTransform(), Matchers.<PTransform<?, ?>>equalTo(replacementTransform));
  // THe tags of the replacement transform are matched to the appropriate PValues of the original
  assertThat(
      replacement.getOutputs().keySet(),
      Matchers.<TupleTag<?>>contains(taggedReplacement.getTag()));
  assertThat(replacement.getOutputs().values(), Matchers.<PValue>contains(originalOutput));
  hierarchy.popNode();
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
private PrimitiveCreate(Create.Values<T> transform, Coder<T> coder) {
  this.transform = transform;
  this.coder = coder;
}
 

import org.apache.beam.sdk.transforms.Create; //导入方法依赖的package包/类
private StreamingFnApiCreate(
    Create.Values<T> transform,
    PCollection<T> originalOutput) {
  this.transform = transform;
  this.originalOutput = originalOutput;
}