Java AbstractStreamOperatorTestHarness.repackageState方法代码示例

import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testScalingDown() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness1 = createRescalingTestSink(outDir, 3, 0);
	testHarness1.setup();
	testHarness1.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness2 = createRescalingTestSink(outDir, 3, 1);
	testHarness2.setup();
	testHarness2.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness3 = createRescalingTestSink(outDir, 3, 2);
	testHarness3.setup();
	testHarness3.open();

	testHarness1.processElement(new StreamRecord<>("test1", 0L));
	checkFs(outDir, 1, 0, 0, 0);

	testHarness2.processElement(new StreamRecord<>("test2", 0L));
	testHarness2.processElement(new StreamRecord<>("test3", 0L));
	testHarness2.processElement(new StreamRecord<>("test4", 0L));
	testHarness2.processElement(new StreamRecord<>("test5", 0L));
	testHarness2.processElement(new StreamRecord<>("test6", 0L));
	checkFs(outDir, 2, 4, 0, 0);

	testHarness3.processElement(new StreamRecord<>("test7", 0L));
	testHarness3.processElement(new StreamRecord<>("test8", 0L));
	checkFs(outDir, 3, 5, 0, 0);

	// intentionally we snapshot them in a not ascending order so that the states are shuffled
	OperatorStateHandles mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(
		testHarness3.snapshot(0, 0),
		testHarness1.snapshot(0, 0),
		testHarness2.snapshot(0, 0)
	);

	// with the above state reshuffling, we expect testHarness4 to take the
	// state of the previous testHarness3 and testHarness1 while testHarness5
	// will take that of the previous testHarness1

	OneInputStreamOperatorTestHarness<String, Object> testHarness4 = createRescalingTestSink(outDir, 2, 0);
	testHarness4.setup();
	testHarness4.initializeState(mergedSnapshot);
	testHarness4.open();

	// we do not have a length file for part-2-0 because bucket part-2-0
	// was not "in-progress", but "pending" (its full content is valid).
	checkFs(outDir, 1, 4, 3, 2);

	OneInputStreamOperatorTestHarness<String, Object> testHarness5 = createRescalingTestSink(outDir, 2, 1);
	testHarness5.setup();
	testHarness5.initializeState(mergedSnapshot);
	testHarness5.open();

	checkFs(outDir, 0, 0, 8, 3);
}
 

import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testScalingUp() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness1 = createRescalingTestSink(outDir, 2, 0);
	testHarness1.setup();
	testHarness1.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness2 = createRescalingTestSink(outDir, 2, 0);
	testHarness2.setup();
	testHarness2.open();

	testHarness1.processElement(new StreamRecord<>("test1", 0L));
	testHarness1.processElement(new StreamRecord<>("test2", 0L));

	checkFs(outDir, 1, 1, 0, 0);

	testHarness2.processElement(new StreamRecord<>("test3", 0L));
	testHarness2.processElement(new StreamRecord<>("test4", 0L));
	testHarness2.processElement(new StreamRecord<>("test5", 0L));

	checkFs(outDir, 2, 3, 0, 0);

	// intentionally we snapshot them in the reverse order so that the states are shuffled
	OperatorStateHandles mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(
		testHarness2.snapshot(0, 0),
		testHarness1.snapshot(0, 0)
	);

	testHarness1 = createRescalingTestSink(outDir, 3, 0);
	testHarness1.setup();
	testHarness1.initializeState(mergedSnapshot);
	testHarness1.open();

	checkFs(outDir, 1, 1, 3, 1);

	testHarness2 = createRescalingTestSink(outDir, 3, 1);
	testHarness2.setup();
	testHarness2.initializeState(mergedSnapshot);
	testHarness2.open();

	checkFs(outDir, 0, 0, 5, 2);

	OneInputStreamOperatorTestHarness<String, Object> testHarness3 = createRescalingTestSink(outDir, 3, 2);
	testHarness3.setup();
	testHarness3.initializeState(mergedSnapshot);
	testHarness3.open();

	checkFs(outDir, 0, 0, 5, 2);

	testHarness1.processElement(new StreamRecord<>("test6", 0));
	testHarness2.processElement(new StreamRecord<>("test6", 0));
	testHarness3.processElement(new StreamRecord<>("test6", 0));

	// 3 for the different tasks
	checkFs(outDir, 3, 0, 5, 2);

	testHarness1.snapshot(1, 0);
	testHarness2.snapshot(1, 0);
	testHarness3.snapshot(1, 0);

	testHarness1.close();
	testHarness2.close();
	testHarness3.close();

	checkFs(outDir, 0, 3, 5, 2);
}
 

import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testSameParallelismWithShufflingStates() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness1 = createRescalingTestSink(outDir, 2, 0, 100);
	testHarness1.setup();
	testHarness1.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness2 = createRescalingTestSink(outDir, 2, 1, 100);
	testHarness2.setup();
	testHarness2.open();

	testHarness1.processElement(new StreamRecord<>("test1", 0L));
	checkFs(outDir, 1, 0, 0, 0);

	testHarness2.processElement(new StreamRecord<>("test2", 0L));
	checkFs(outDir, 2, 0, 0, 0);

	// intentionally we snapshot them in the reverse order so that the states are shuffled
	OperatorStateHandles mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(
		testHarness2.snapshot(0, 0),
		testHarness1.snapshot(0, 0)
	);

	checkFs(outDir, 2, 0, 0, 0);

	// this will not be included in any checkpoint so it can be cleaned up (although we do not)
	testHarness2.processElement(new StreamRecord<>("test3", 0L));
	checkFs(outDir, 3, 0, 0, 0);

	testHarness1 = createRescalingTestSink(outDir, 2, 0, 100);
	testHarness1.setup();
	testHarness1.initializeState(mergedSnapshot);
	testHarness1.open();

	// the one in-progress will be the one assigned to the next instance,
	// the other is the test3 which is just not cleaned up
	checkFs(outDir, 2, 0, 1, 1);

	testHarness2 = createRescalingTestSink(outDir, 2, 1, 100);
	testHarness2.setup();
	testHarness2.initializeState(mergedSnapshot);
	testHarness2.open();

	checkFs(outDir, 1, 0, 2, 2);

	testHarness1.close();
	testHarness2.close();

	// the 1 in-progress can be discarded.
	checkFs(outDir, 1, 0, 2, 2);
}
 

import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testScalingDown() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness1 = createRescalingTestSink(outDir, 3, 0, 100);
	testHarness1.setup();
	testHarness1.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness2 = createRescalingTestSink(outDir, 3, 1, 100);
	testHarness2.setup();
	testHarness2.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness3 = createRescalingTestSink(outDir, 3, 2, 100);
	testHarness3.setup();
	testHarness3.open();

	testHarness1.processElement(new StreamRecord<>("test1", 0L));
	checkFs(outDir, 1, 0, 0, 0);

	testHarness2.processElement(new StreamRecord<>("test2", 0L));
	checkFs(outDir, 2, 0, 0, 0);

	testHarness3.processElement(new StreamRecord<>("test3", 0L));
	testHarness3.processElement(new StreamRecord<>("test4", 0L));
	checkFs(outDir, 4, 0, 0, 0);

	// intentionally we snapshot them in the reverse order so that the states are shuffled
	OperatorStateHandles mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(
		testHarness3.snapshot(0, 0),
		testHarness1.snapshot(0, 0),
		testHarness2.snapshot(0, 0)
	);

	testHarness1 = createRescalingTestSink(outDir, 2, 0, 100);
	testHarness1.setup();
	testHarness1.initializeState(mergedSnapshot);
	testHarness1.open();

	checkFs(outDir, 1, 0, 3, 3);

	testHarness2 = createRescalingTestSink(outDir, 2, 1, 100);
	testHarness2.setup();
	testHarness2.initializeState(mergedSnapshot);
	testHarness2.open();

	checkFs(outDir, 0, 0, 4, 4);
}
 

import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testScalingUp() throws Exception {
	final File outDir = tempFolder.newFolder();

	OneInputStreamOperatorTestHarness<String, Object> testHarness1 = createRescalingTestSink(outDir, 2, 0, 100);
	testHarness1.setup();
	testHarness1.open();

	OneInputStreamOperatorTestHarness<String, Object> testHarness2 = createRescalingTestSink(outDir, 2, 0, 100);
	testHarness2.setup();
	testHarness2.open();

	testHarness1.processElement(new StreamRecord<>("test1", 1L));
	testHarness1.processElement(new StreamRecord<>("test2", 1L));

	checkFs(outDir, 2, 0, 0, 0);

	testHarness2.processElement(new StreamRecord<>("test3", 1L));
	testHarness2.processElement(new StreamRecord<>("test4", 1L));
	testHarness2.processElement(new StreamRecord<>("test5", 1L));

	checkFs(outDir, 5, 0, 0, 0);

	// intentionally we snapshot them in the reverse order so that the states are shuffled
	OperatorStateHandles mergedSnapshot = AbstractStreamOperatorTestHarness.repackageState(
		testHarness2.snapshot(0, 0),
		testHarness1.snapshot(0, 0)
	);

	testHarness1 = createRescalingTestSink(outDir, 3, 0, 100);
	testHarness1.setup();
	testHarness1.initializeState(mergedSnapshot);
	testHarness1.open();

	checkFs(outDir, 2, 0, 3, 3);

	testHarness2 = createRescalingTestSink(outDir, 3, 1, 100);
	testHarness2.setup();
	testHarness2.initializeState(mergedSnapshot);
	testHarness2.open();

	checkFs(outDir, 0, 0, 5, 5);

	OneInputStreamOperatorTestHarness<String, Object> testHarness3 = createRescalingTestSink(outDir, 3, 2, 100);
	testHarness3.setup();
	testHarness3.initializeState(mergedSnapshot);
	testHarness3.open();

	checkFs(outDir, 0, 0, 5, 5);

	testHarness1.processElement(new StreamRecord<>("test6", 0));
	testHarness2.processElement(new StreamRecord<>("test6", 0));
	testHarness3.processElement(new StreamRecord<>("test6", 0));

	checkFs(outDir, 3, 0, 5, 5);

	testHarness1.snapshot(1, 0);
	testHarness2.snapshot(1, 0);
	testHarness3.snapshot(1, 0);

	testHarness1.close();
	testHarness2.close();
	testHarness3.close();

	checkFs(outDir, 0, 3, 5, 5);
}
 

import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testScalingDown() throws Exception {
	S sink1 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> testHarness1 =
		new OneInputStreamOperatorTestHarness<>(sink1, 10, 2, 0);
	testHarness1.open();

	S sink2 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> testHarness2 =
		new OneInputStreamOperatorTestHarness<>(sink2, 10, 2, 1);
	testHarness2.open();

	int elementCounter = 1;
	int snapshotCount = 0;

	for (int x = 0; x < 10; x++) {
		testHarness1.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	for (int x = 0; x < 11; x++) {
		testHarness2.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	// snapshot at checkpoint 0 for testHarness1 and testHarness 2
	OperatorStateHandles snapshot1 = testHarness1.snapshot(snapshotCount, 0);
	OperatorStateHandles snapshot2 = testHarness2.snapshot(snapshotCount, 0);

	// merge the two partial states
	OperatorStateHandles mergedSnapshot = AbstractStreamOperatorTestHarness
		.repackageState(snapshot1, snapshot2);

	testHarness1.close();
	testHarness2.close();

	// and create a third instance that operates alone but
	// has the merged state of the previous 2 instances

	S sink3 = createSink();
	OneInputStreamOperatorTestHarness<IN, IN> mergedTestHarness =
		new OneInputStreamOperatorTestHarness<>(sink3, 10, 1, 0);

	mergedTestHarness.setup();
	mergedTestHarness.initializeState(mergedSnapshot);
	mergedTestHarness.open();

	for (int x = 0; x < 12; x++) {
		mergedTestHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0)));
		elementCounter++;
	}

	snapshotCount++;
	mergedTestHarness.snapshot(snapshotCount, 1);
	mergedTestHarness.notifyOfCompletedCheckpoint(snapshotCount);

	verifyResultsWhenReScaling(sink3, 1, 33);
	mergedTestHarness.close();
}