Java TestPipeline类代码示例

import com.google.cloud.dataflow.sdk.testing.TestPipeline; //导入依赖的package包/类
@Test
public void test_reading() throws Exception {
    final File file =
            new File(getClass().getResource("/sample.csv").toURI());
    assertThat(file.exists()).isTrue();

    final Pipeline pipeline = TestPipeline.create();

    final PCollection<String> output =
            pipeline.apply(Read.from(CsvWithHeaderFileSource.from(file.getAbsolutePath())));

    DataflowAssert
            .that(output)
            .containsInAnyOrder("a:1, b:2, c:3, ", "a:4, b:5, c:6, ");

    pipeline.run();
}
 

import com.google.cloud.dataflow.sdk.testing.TestPipeline; //导入依赖的package包/类
@Test
public void parseBooks_returnsNgramsCounts() {
  // Arrange
  Pipeline p = TestPipeline.create();
  PCollection<String> input = p.apply(Create.of(testFile));

  // Act
  PCollection<KV<String, Integer>> output = LoadBooks.applyPipelineToParseBooks(input);

  // Assert
  DataflowAssert.that(output)
      .containsInAnyOrder(
          KV.of("despatch when art", 10),
          KV.of("despatch when came", 10),
          KV.of("despatch when published", 12),
          KV.of("despatch where was", 10),
          KV.of("despatch which made", 45),
          // There are two entries for "despatch which addressed".
          // Each entry has a different part of speech for "addressed".
          KV.of("despatch which addressed", 12 + 46),
          KV.of("despatch which admitted", 13),
          KV.of("despatch which allow", 14),
          KV.of("despatch which announced", 50),
          KV.of("despatch which answer", 32));
}
 

import com.google.cloud.dataflow.sdk.testing.TestPipeline; //导入依赖的package包/类
@Test
public void testCSVToMapLineCombineFn() {

    final String[] csv = new String[] {
            "000,Tokyo,120",
            "001,Osaka,100",
            "002,Kyoto,140"
    };
    final List<String> csvlist = Arrays.asList(csv);

    Pipeline p = TestPipeline.create();
    PCollection<String> maplines = p.apply(Create.of(csvlist)).setCoder(StringUtf8Coder.of());
    PCollectionView<Map<String,String>> mapview = maplines.apply(Combine.globally(new CSVToMapLineCombineFn()).asSingletonView());

    final String[] dummy = new String[] {
            "000",
            "001",
            "002"
    };
    List<String> dummylist = Arrays.asList(dummy);
    DoFnTester<String,String> fnTester = DoFnTester.of(new AAA(mapview));
    fnTester.setSideInputInGlobalWindow(mapview, csvlist);

    //dummylines.apply(ParDo.of(fnTester));
    List<String> results = fnTester.processBatch(dummylist);
    System.out.println(results);

    //p.apply()
}
 

import com.google.cloud.dataflow.sdk.testing.TestPipeline; //导入依赖的package包/类
/** Example test that tests a PTransform by using an in-memory input and inspecting the output. */
@Test
@Category(RunnableOnService.class)
public void testCountWords() throws Exception {
  Pipeline p = TestPipeline.create();

  PCollection<String> input = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));

  PCollection<String> output = input.apply(new CountWords())
    .apply(ParDo.of(new FormatAsTextFn()));

  DataflowAssert.that(output).containsInAnyOrder(COUNTS_ARRAY);
  p.run();
}
 

import com.google.cloud.dataflow.sdk.testing.TestPipeline; //导入依赖的package包/类
@Test(expectedExceptions = UserException.class)
public void testUserExceptionUnwrapping(){
    Pipeline p = TestPipeline.create(); // use regular pipeline as Spark doesn't return user exceptions yet
    PCollection<String> pstrings = p.apply(Create.of(Lists.newArrayList("Some", "Values")));
    pstrings.apply(DataflowUtils.throwException(new UserException("fail")));
    DataflowCommandLineProgram.runPipeline(p);
}
 

import com.google.cloud.dataflow.sdk.testing.TestPipeline; //导入依赖的package包/类
@Before
public void setUp() {
  pipeline = TestPipeline.create();
}