Java Create类代码示例

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testUnwritableRemoveContainerPipeline() throws Exception {

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/dataDirectory2").toURI().toString());

    final File root = new File(getClass().getResource("/dataDirectory2").toURI());

    assumeTrue(root.setReadOnly());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(CONTAINER_KV))
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), false)));

    PAssert.that(pCollection).empty();

    pipeline.run();
    root.setWritable(true);
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testUnwritableAddContainerPipeline() throws Exception {

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/dataDirectory2").toURI().toString());

    final File root = new File(getClass().getResource("/dataDirectory2").toURI());

    assumeTrue(root.setReadOnly());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(CONTAINER_KV))
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), true)));

    PAssert.that(pCollection).empty();

    pipeline.run();
    root.setWritable(true);
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
public static void main(String[] args) {
  Pipeline p = Pipeline.create(
      PipelineOptionsFactory.fromArgs(args).withValidation().create());

  p.apply(Create.of("Hello", "World"))
  .apply(MapElements.via(new SimpleFunction<String, String>() {
    @Override
    public String apply(String input) {
      return input.toUpperCase();
    }
  }))
  .apply(ParDo.of(new DoFn<String, Void>() {
    @ProcessElement
    public void processElement(ProcessContext c)  {
      LOG.info(c.element());
    }
  }));

  p.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testCachePipeline() throws Exception {

    final KV<String, String> kv = KV.of("trellis:repository/resource", null);

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/root").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new CacheWriter(dataConfiguration)));

    PAssert.that(pCollection).containsInAnyOrder(asList(kv));

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testUnableToCachePipeline() throws Exception {

    final KV<String, String> kv = KV.of("trellis:repository/some-other-resource", null);

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/root").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new CacheWriter(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDirectoryPipeline() throws Exception {

    final KV<String, String> kv = KV.of("trellis:repository/resource", null);

    final Map<String, String> dataConfiguration = singletonMap("foo",
            getClass().getResource("/root").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new CacheWriter(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final Map<String, String> dataConfiguration = singletonMap("repository",
            getClass().getResource("/dataDirectory").toURI().toString());

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), false)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final Map<String, String> dataConfiguration = singletonMap("repository", "http://localhost/");

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new EventProcessor(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final String dataConfiguration = getClass().getResource("/dataDirectory").toURI().toString();

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new BeamProcessor(dataConfiguration, LDP.PreferContainment.getIRIString(), false)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
@Category(NeedsRunner.class)
public void testInvalidDataPipeline() throws Exception {

    final String dataset = "<trellis:repository/resource> " +
        "<http://purl.org/dc/terms/subject> <trellis:repository/resource/member> " +
        "<http://www.w3.org/ns/ldp#PreferConta";
    final KV<String, String> kv = KV.of("trellis:repository/resource", dataset);

    final String dataConfiguration = "http://localhost/";

    final PCollection<KV<String, String>> pCollection = pipeline
        .apply("Create", Create.of(kv))
        .apply(ParDo.of(new EventProcessor(dataConfiguration)));

    PAssert.that(pCollection).empty();

    pipeline.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
public void testTransformer() {
  // Read file
  InputStream inputStream =
      this.getClass().getResourceAsStream("/dummy-log");
  // Convert to JSON string
  String json = readStream(inputStream);

  LogAggregator.LogTransformer transformer = new LogAggregator.LogTransformer();

  List<String> inputs = Arrays.asList(json, json, json);
  PCollection<String> inputCollection = pipeline.apply(Create.of(inputs));
  PCollection<TableRow> outputCollection = inputCollection.apply(transformer);
  PAssert.that(outputCollection).satisfies(new HasColumnsCheckerFn());
  pipeline.run().waitUntilFinish();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
public void testMultiOutputOverrideNonCrashing() throws Exception {
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setRunner(DataflowRunner.class);
  Pipeline pipeline = Pipeline.create(options);

  TupleTag<Integer> mainOutputTag = new TupleTag<Integer>() {};
  TupleTag<Integer> sideOutputTag = new TupleTag<Integer>() {};

  DummyStatefulDoFn fn = new DummyStatefulDoFn();
  pipeline
      .apply(Create.of(KV.of(1, 2)))
      .apply(ParDo.of(fn).withOutputTags(mainOutputTag, TupleTagList.of(sideOutputTag)));

  DataflowRunner runner = DataflowRunner.fromOptions(options);
  runner.replaceTransforms(pipeline);
  assertThat(findBatchStatefulDoFn(pipeline), equalTo((DoFn) fn));
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
@Test
public void testPassThroughThenCleanup() throws Exception {

  PCollection<Integer> output =
      p.apply(Create.of(1, 2, 3))
          .apply(
              new PassThroughThenCleanup<Integer>(
                  new PassThroughThenCleanup.CleanupOperation() {
                    @Override
                    void cleanup(PassThroughThenCleanup.ContextContainer c) throws Exception {
                      // no-op
                    }
                  },
                  p.apply("Create1", Create.of("")).apply(View.<String>asSingleton())));

  PAssert.that(output).containsInAnyOrder(1, 2, 3);

  p.run();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
/** Test that the example runs. */
@Test
@Category(ValidatesRunner.class)
public void testTfIdf() throws Exception {

  pipeline.getCoderRegistry().registerCoderForClass(URI.class, StringDelegateCoder.of(URI.class));

  PCollection<KV<String, KV<URI, Double>>> wordToUriAndTfIdf = pipeline
      .apply(Create.of(
          KV.of(new URI("x"), "a b c d"),
          KV.of(new URI("y"), "a b c"),
          KV.of(new URI("z"), "a m n")))
      .apply(new TfIdf.ComputeTfIdf());

  PCollection<String> words = wordToUriAndTfIdf
      .apply(Keys.<String>create())
      .apply(Distinct.<String>create());

  PAssert.that(words).containsInAnyOrder(Arrays.asList("a", "m", "n", "b", "c", "d"));

  pipeline.run().waitUntilFinish();
}
 

import org.apache.beam.sdk.transforms.Create; //导入依赖的package包/类
/**
 * Tests that a {@link DoFn} that mutates an output with a good equals() fails in the
 * {@link DirectRunner}.
 */
@Test
public void testMutatingOutputThenOutputDoFnError() throws Exception {
  Pipeline pipeline = getPipeline();

  pipeline
      .apply(Create.of(42))
      .apply(ParDo.of(new DoFn<Integer, List<Integer>>() {
        @ProcessElement
        public void processElement(ProcessContext c) {
          List<Integer> outputList = Arrays.asList(1, 2, 3, 4);
          c.output(outputList);
          outputList.set(0, 37);
          c.output(outputList);
        }
      }));

  thrown.expect(IllegalMutationException.class);
  thrown.expectMessage("output");
  thrown.expectMessage("must not be mutated");
  pipeline.run();
}