本文整理汇总了Java中org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness.initializeState方法的典型用法代码示例。如果您正苦于以下问题:Java AbstractStreamOperatorTestHarness.initializeState方法的具体用法?Java AbstractStreamOperatorTestHarness.initializeState怎么用?Java AbstractStreamOperatorTestHarness.initializeState使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness
的用法示例。
在下文中一共展示了AbstractStreamOperatorTestHarness.initializeState方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testRestoreFromEmptyStateWithPartitions
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
/**
* Test restoring from an empty state taken using Flink 1.2, when some partitions could be
* found for topics.
*/
@Test
public void testRestoreFromEmptyStateWithPartitions() throws Exception {
final List<KafkaTopicPartition> partitions = new ArrayList<>(PARTITION_STATE.keySet());
final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(partitions);
StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator =
new StreamSource<>(consumerFunction);
final AbstractStreamOperatorTestHarness<String> testHarness =
new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
testHarness.setup();
// restore state from binary snapshot file
testHarness.initializeState(
OperatorSnapshotUtil.readStateHandle(
OperatorSnapshotUtil.getResourceFilename("kafka-consumer-migration-test-flink1.2-empty-state-snapshot")));
testHarness.open();
// the expected state in "kafka-consumer-migration-test-flink1.2-empty-state-snapshot";
// since the state is empty, the consumer should reflect on the startup mode to determine start offsets.
final HashMap<KafkaTopicPartition, Long> expectedSubscribedPartitionsWithStartOffsets = new HashMap<>();
for (KafkaTopicPartition partition : PARTITION_STATE.keySet()) {
expectedSubscribedPartitionsWithStartOffsets.put(partition, KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
}
// assert that there are partitions and is identical to expected list
assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
assertTrue(!consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
Assert.assertEquals(expectedSubscribedPartitionsWithStartOffsets, consumerFunction.getSubscribedPartitionsToStartOffsets());
assertTrue(consumerFunction.getRestoredState() == null);
consumerOperator.close();
consumerOperator.cancel();
}
示例2: testRestoreFromEmptyStateNoPartitions
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
/**
* Test restoring from an legacy empty state, when no partitions could be found for topics.
*/
@Test
public void testRestoreFromEmptyStateNoPartitions() throws Exception {
final DummyFlinkKafkaConsumer<String> consumerFunction =
new DummyFlinkKafkaConsumer<>(Collections.<KafkaTopicPartition>emptyList());
StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
final AbstractStreamOperatorTestHarness<String> testHarness =
new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
testHarness.setup();
// restore state from binary snapshot file
testHarness.initializeState(
OperatorSnapshotUtil.readStateHandle(
OperatorSnapshotUtil.getResourceFilename("kafka-consumer-migration-test-flink1.2-empty-state-snapshot")));
testHarness.open();
// assert that no partitions were found and is empty
assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
// assert that no state was restored
assertTrue(consumerFunction.getRestoredState() == null);
consumerOperator.close();
consumerOperator.cancel();
}
示例3: testRestore
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
/**
* Test restoring from a non-empty state taken using Flink 1.2, when some partitions could be
* found for topics.
*/
@Test
public void testRestore() throws Exception {
final List<KafkaTopicPartition> partitions = new ArrayList<>(PARTITION_STATE.keySet());
final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(partitions);
StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator =
new StreamSource<>(consumerFunction);
final AbstractStreamOperatorTestHarness<String> testHarness =
new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
testHarness.setup();
// restore state from binary snapshot file
testHarness.initializeState(
OperatorSnapshotUtil.readStateHandle(
OperatorSnapshotUtil.getResourceFilename("kafka-consumer-migration-test-flink1.2-snapshot")));
testHarness.open();
// assert that there are partitions and is identical to expected list
assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
assertTrue(!consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
// on restore, subscribedPartitionsToStartOffsets should be identical to the restored state
Assert.assertEquals(PARTITION_STATE, consumerFunction.getSubscribedPartitionsToStartOffsets());
// assert that state is correctly restored from legacy checkpoint
assertTrue(consumerFunction.getRestoredState() != null);
Assert.assertEquals(PARTITION_STATE, consumerFunction.getRestoredState());
consumerOperator.close();
consumerOperator.cancel();
}
示例4: testUDFReturningNull
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testUDFReturningNull() throws Exception {
TestUserFunction userFunction = new TestUserFunction(null);
AbstractStreamOperatorTestHarness<Integer> testHarness =
new AbstractStreamOperatorTestHarness<>(new StreamMap<>(userFunction), 1, 1, 0);
testHarness.open();
OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
testHarness.initializeState(snapshot);
Assert.assertTrue(userFunction.isRestored());
}
示例5: testUDFReturningEmpty
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testUDFReturningEmpty() throws Exception {
TestUserFunction userFunction = new TestUserFunction(Collections.<Integer>emptyList());
AbstractStreamOperatorTestHarness<Integer> testHarness =
new AbstractStreamOperatorTestHarness<>(new StreamMap<>(userFunction), 1, 1, 0);
testHarness.open();
OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
testHarness.initializeState(snapshot);
Assert.assertTrue(userFunction.isRestored());
}
示例6: testUDFReturningData
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testUDFReturningData() throws Exception {
TestUserFunction userFunction = new TestUserFunction(Arrays.asList(1, 2, 3));
AbstractStreamOperatorTestHarness<Integer> testHarness =
new AbstractStreamOperatorTestHarness<>(new StreamMap<>(userFunction), 1, 1, 0);
testHarness.open();
OperatorStateHandles snapshot = testHarness.snapshot(0L, 0L);
testHarness.initializeState(snapshot);
Assert.assertTrue(userFunction.isRestored());
}
示例7: testNullCheckpoint
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testNullCheckpoint() throws Exception {
final int numElements = 20;
PipelineOptions options = PipelineOptionsFactory.create();
TestCountingSource source = new TestCountingSource(numElements) {
@Override
public Coder<CounterMark> getCheckpointMarkCoder() {
return null;
}
};
UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark> flinkWrapper =
new UnboundedSourceWrapper<>("stepName", options, source, numSplits);
StreamSource<
WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark>>
sourceOperator = new StreamSource<>(flinkWrapper);
AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
testHarness =
new AbstractStreamOperatorTestHarness<>(
sourceOperator,
numTasks /* max parallelism */,
numTasks /* parallelism */,
0 /* subtask index */);
testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
testHarness.open();
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
UnboundedSourceWrapper<
KV<Integer, Integer>, TestCountingSource.CounterMark> restoredFlinkWrapper =
new UnboundedSourceWrapper<>(
"stepName", options, new TestCountingSource(numElements), numSplits);
StreamSource<
WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>,
UnboundedSourceWrapper<KV<Integer, Integer>, TestCountingSource.CounterMark>>
restoredSourceOperator =
new StreamSource<>(restoredFlinkWrapper);
// set parallelism to 1 to ensure that our testing operator gets all checkpointed state
AbstractStreamOperatorTestHarness<WindowedValue<ValueWithRecordId<KV<Integer, Integer>>>>
restoredTestHarness =
new AbstractStreamOperatorTestHarness<>(
restoredSourceOperator,
numTasks /* max parallelism */,
1 /* parallelism */,
0 /* subtask index */);
restoredTestHarness.setup();
restoredTestHarness.initializeState(snapshot);
restoredTestHarness.open();
// when the source checkpointed a null we don't re-initialize the splits, that is we
// will have no splits.
assertEquals(0, restoredFlinkWrapper.getLocalSplitSources().size());
}
示例8: testCheckpointing
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testCheckpointing() throws Exception {
source.autoAck = false;
StreamSource<String, RMQSource<String>> src = new StreamSource<>(source);
AbstractStreamOperatorTestHarness<String> testHarness =
new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
testHarness.open();
sourceThread.start();
Thread.sleep(5);
final Random random = new Random(System.currentTimeMillis());
int numSnapshots = 50;
long previousSnapshotId;
long lastSnapshotId = 0;
long totalNumberOfAcks = 0;
for (int i = 0; i < numSnapshots; i++) {
long snapshotId = random.nextLong();
OperatorStateHandles data;
synchronized (DummySourceContext.lock) {
data = testHarness.snapshot(snapshotId, System.currentTimeMillis());
previousSnapshotId = lastSnapshotId;
lastSnapshotId = messageId;
}
// let some time pass
Thread.sleep(5);
// check if the correct number of messages have been snapshotted
final long numIds = lastSnapshotId - previousSnapshotId;
RMQTestSource sourceCopy = new RMQTestSource();
StreamSource<String, RMQTestSource> srcCopy = new StreamSource<>(sourceCopy);
AbstractStreamOperatorTestHarness<String> testHarnessCopy =
new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
testHarnessCopy.setup();
testHarnessCopy.initializeState(data);
testHarnessCopy.open();
ArrayDeque<Tuple2<Long, Set<String>>> deque = sourceCopy.getRestoredState();
Set<String> messageIds = deque.getLast().f1;
assertEquals(numIds, messageIds.size());
if (messageIds.size() > 0) {
assertTrue(messageIds.contains(Long.toString(lastSnapshotId)));
}
// check if the messages are being acknowledged and the transaction committed
synchronized (DummySourceContext.lock) {
source.notifyCheckpointComplete(snapshotId);
}
totalNumberOfAcks += numIds;
}
Mockito.verify(source.channel, Mockito.times((int) totalNumberOfAcks)).basicAck(Mockito.anyLong(), Mockito.eq(false));
Mockito.verify(source.channel, Mockito.times(numSnapshots)).txCommit();
}
示例9: testFunctionRestore
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testFunctionRestore() throws Exception {
String testBasePath = hdfsURI + "/" + UUID.randomUUID() + "/";
org.apache.hadoop.fs.Path path = null;
long fileModTime = Long.MIN_VALUE;
for (int i = 0; i < 1; i++) {
Tuple2<org.apache.hadoop.fs.Path, String> file = createFileAndFillWithData(testBasePath, "file", i, "This is test line.");
path = file.f0;
fileModTime = hdfs.getFileStatus(file.f0).getModificationTime();
}
TextInputFormat format = new TextInputFormat(new Path(testBasePath));
final ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> src =
new StreamSource<>(monitoringFunction);
final AbstractStreamOperatorTestHarness<TimestampedFileInputSplit> testHarness =
new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
testHarness.open();
final Throwable[] error = new Throwable[1];
final OneShotLatch latch = new OneShotLatch();
final DummySourceContext sourceContext = new DummySourceContext() {
@Override
public void collect(TimestampedFileInputSplit element) {
latch.trigger();
}
};
// run the source asynchronously
Thread runner = new Thread() {
@Override
public void run() {
try {
monitoringFunction.run(sourceContext);
}
catch (Throwable t) {
t.printStackTrace();
error[0] = t;
}
}
};
runner.start();
// first condition for the source to have updated its state: emit at least one element
if (!latch.isTriggered()) {
latch.await();
}
// second condition for the source to have updated its state: it's not on the lock anymore,
// this means it has processed all the splits and updated its state.
synchronized (sourceContext.getCheckpointLock()) {}
OperatorStateHandles snapshot = testHarness.snapshot(0, 0);
monitoringFunction.cancel();
runner.join();
testHarness.close();
final ContinuousFileMonitoringFunction<String> monitoringFunctionCopy =
new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> srcCopy =
new StreamSource<>(monitoringFunctionCopy);
AbstractStreamOperatorTestHarness<TimestampedFileInputSplit> testHarnessCopy =
new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
testHarnessCopy.initializeState(snapshot);
testHarnessCopy.open();
Assert.assertNull(error[0]);
Assert.assertEquals(fileModTime, monitoringFunctionCopy.getGlobalModificationTime());
hdfs.delete(path, false);
}
示例10: testCheckpointAndRestore
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testCheckpointAndRestore() {
try {
final int numElements = 10000;
List<Integer> data = new ArrayList<Integer>(numElements);
List<Integer> result = new ArrayList<Integer>(numElements);
for (int i = 0; i < numElements; i++) {
data.add(i);
}
final FromElementsFunction<Integer> source = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
StreamSource<Integer, FromElementsFunction<Integer>> src = new StreamSource<>(source);
AbstractStreamOperatorTestHarness<Integer> testHarness =
new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
testHarness.open();
final SourceFunction.SourceContext<Integer> ctx = new ListSourceContext<Integer>(result, 2L);
final Throwable[] error = new Throwable[1];
// run the source asynchronously
Thread runner = new Thread() {
@Override
public void run() {
try {
source.run(ctx);
}
catch (Throwable t) {
error[0] = t;
}
}
};
runner.start();
// wait for a bit
Thread.sleep(1000);
// make a checkpoint
List<Integer> checkpointData = new ArrayList<>(numElements);
OperatorStateHandles handles = null;
synchronized (ctx.getCheckpointLock()) {
handles = testHarness.snapshot(566, System.currentTimeMillis());
checkpointData.addAll(result);
}
// cancel the source
source.cancel();
runner.join();
// check for errors
if (error[0] != null) {
System.err.println("Error in asynchronous source runner");
error[0].printStackTrace();
fail("Error in asynchronous source runner");
}
final FromElementsFunction<Integer> sourceCopy = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
StreamSource<Integer, FromElementsFunction<Integer>> srcCopy = new StreamSource<>(sourceCopy);
AbstractStreamOperatorTestHarness<Integer> testHarnessCopy =
new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
testHarnessCopy.setup();
testHarnessCopy.initializeState(handles);
testHarnessCopy.open();
// recovery run
SourceFunction.SourceContext<Integer> newCtx = new ListSourceContext<>(checkpointData);
sourceCopy.run(newCtx);
assertEquals(data, checkpointData);
}
catch (Exception e) {
e.printStackTrace();
fail(e.getMessage());
}
}
示例11: testMonitoringSourceRestore
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
@Test
public void testMonitoringSourceRestore() throws Exception {
File testFolder = tempFolder.newFolder();
Long expectedModTime = Long.parseLong("1493116191000");
TextInputFormat format = new TextInputFormat(new Path(testFolder.getAbsolutePath()));
final ContinuousFileMonitoringFunction<String> monitoringFunction =
new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
StreamSource<TimestampedFileInputSplit, ContinuousFileMonitoringFunction<String>> src =
new StreamSource<>(monitoringFunction);
final AbstractStreamOperatorTestHarness<TimestampedFileInputSplit> testHarness =
new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
testHarness.setup();
OperatorStateHandles operatorStateHandles = OperatorSnapshotUtil.readStateHandle(
OperatorSnapshotUtil.getResourceFilename(
"monitoring-function-migration-test-1493116191000-flink1.2-snapshot"));
testHarness.initializeState(operatorStateHandles);
testHarness.open();
Assert.assertEquals((long) expectedModTime, monitoringFunction.getGlobalModificationTime());
}
示例12: restoreFromSnapshot
import org.apache.flink.streaming.util.AbstractStreamOperatorTestHarness; //导入方法依赖的package包/类
/**
* Restore from a snapshot taken with an older Flink version.
*
* @param testHarness the test harness to restore the snapshot to.
* @param snapshotPath the absolute path to the snapshot.
* @param snapshotFlinkVersion the Flink version of the snapshot.
* @throws Exception
*/
public static void restoreFromSnapshot(
AbstractStreamOperatorTestHarness<?> testHarness,
String snapshotPath,
MigrationVersion snapshotFlinkVersion) throws Exception {
testHarness.initializeState(OperatorSnapshotUtil.readStateHandle(snapshotPath));
}