Java DataSet.union方法代码示例

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testSupportForDataAndEnumSerialization() throws Exception {
	/**
	 * Test support for Date and enum serialization
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<PojoWithDateAndEnum> ds = env.generateSequence(0,2).map(new Mapper1());
	ds = ds.union(CollectionDataSets.getPojoWithDateAndEnum(env));

	DataSet<String> res = ds.groupBy("group").reduceGroup(new GroupReducer1());

	List<String> result = res.collect();

	String expected = "ok\nok";

	compareResultAsText(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testUnion2IdenticalDataSets() throws Exception {
	/*
	 * Union of 2 Same Data Sets
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> unionDs = ds.union(CollectionDataSets.get3TupleDataSet(env));

	List<Tuple3<Integer, Long, String>> result = unionDs.collect();

	String expected = FULL_TUPLE_3_STRING + FULL_TUPLE_3_STRING;

	compareResultAsTuples(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testSupportForDataAndEnumSerialization() throws Exception {
	/**
	 * Test support for Date and enum serialization
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<PojoWithDateAndEnum> ds = env.generateSequence(0, 2).map(new Mapper1());
	ds = ds.union(CollectionDataSets.getPojoWithDateAndEnum(env));

	DataSet<String> res = ds.groupBy("group").reduceGroup(new GroupReducer1());

	List<String> result = res.collect();

	String expected = "ok\nok";

	compareResultAsText(result, expected);
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Test
public void testUnionReplacement() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<String> input1 = env.fromElements("test1");
		DataSet<String> input2 = env.fromElements("test2");

		DataSet<String> union = input1.union(input2);

		union.output(new DiscardingOutputFormat<String>());
		union.output(new DiscardingOutputFormat<String>());

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		JobGraphGenerator jobGen = new JobGraphGenerator();
		jobGen.compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.api.java.DataSet; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void translateNode(
    PTransform<PCollectionList<T>, PCollection<T>> transform,
    FlinkBatchTranslationContext context) {

  Map<TupleTag<?>, PValue> allInputs = context.getInputs(transform);
  DataSet<WindowedValue<T>> result = null;

  if (allInputs.isEmpty()) {

    // create an empty dummy source to satisfy downstream operations
    // we cannot create an empty source in Flink, therefore we have to
    // add the flatMap that simply never forwards the single element
    DataSource<String> dummySource =
        context.getExecutionEnvironment().fromElements("dummy");
    result = dummySource.flatMap(new FlatMapFunction<String, WindowedValue<T>>() {
      @Override
      public void flatMap(String s, Collector<WindowedValue<T>> collector) throws Exception {
        // never return anything
      }
    }).returns(
        new CoderTypeInformation<>(
            WindowedValue.getFullCoder(
                (Coder<T>) VoidCoder.of(),
                GlobalWindow.Coder.INSTANCE)));
  } else {
    for (PValue taggedPc : allInputs.values()) {
      checkArgument(
          taggedPc instanceof PCollection,
          "Got non-PCollection input to flatten: %s of type %s",
          taggedPc,
          taggedPc.getClass().getSimpleName());
      PCollection<T> collection = (PCollection<T>) taggedPc;
      DataSet<WindowedValue<T>> current = context.getInputDataSet(collection);
      if (result == null) {
        result = current;
      } else {
        result = result.union(current);
      }
    }
  }

  // insert a dummy filter, there seems to be a bug in Flink
  // that produces duplicate elements after the union in some cases
  // if we don't
  result = result.filter(new FilterFunction<WindowedValue<T>>() {
    @Override
    public boolean filter(WindowedValue<T> tWindowedValue) throws Exception {
      return true;
    }
  }).name("UnionFixFilter");
  context.setOutputDataSet(context.getOutput(transform), result);
}