Java StreamSink類代碼示例

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Test(timeout = 5000)
public void testThrowErrorOnSnapshot() throws Exception {
	TestCassandraSink casSinkFunc = new TestCassandraSink();

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(casSinkFunc));

	testHarness.open();

	Exception cause = new RuntimeException();
	casSinkFunc.setResultFuture(ResultSetFutures.fromCompletableFuture(FutureUtils.getFailedFuture(cause)));

	casSinkFunc.invoke("hello");

	try {
		testHarness.snapshot(123L, 123L);

		Assert.fail();
	} catch (Exception e) {
		Assert.assertTrue(e.getCause() instanceof IOException);
		Assert.assertEquals(cause, e.getCause().getCause());
		Assert.assertEquals(0, casSinkFunc.getNumOfPendingRecords());
	}

	testHarness.close();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
private OneInputStreamOperatorTestHarness<Integer, Object> createTestHarness(
	String topic,
	int maxParallelism,
	int parallelism,
	int subtaskIndex) throws Exception {
	Properties properties = createProperties();

	FlinkKafkaProducer011<Integer> kafkaProducer = new FlinkKafkaProducer011<>(
		topic,
		integerKeyedSerializationSchema,
		properties,
		FlinkKafkaProducer011.Semantic.EXACTLY_ONCE);

	return new OneInputStreamOperatorTestHarness<>(
		new StreamSink<>(kafkaProducer),
		maxParallelism,
		parallelism,
		subtaskIndex,
		IntSerializer.INSTANCE);
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/**
 * This test is meant to assure that testAtLeastOnceSink is valid by testing that if flushing is disabled,
 * the snapshot method does indeed finishes without waiting for pending requests;
 * we set a timeout because the test will not finish if the logic is broken.
 */
@Test(timeout = 5000)
public void testDoesNotWaitForPendingRequestsIfFlushingDisabled() throws Exception {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new DummyRetryFailureHandler());
	sink.disableFlushOnCheckpoint(); // disable flushing

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and let bulk request succeed
	sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record")));
	testHarness.processElement(new StreamRecord<>("msg-1"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(ActionRequest.class));

	// the snapshot should not block even though we haven't flushed the bulk request
	testHarness.snapshot(1L, 1000L);

	testHarness.close();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/**
 * This test is meant to assure that testAtLeastOnceProducer is valid by testing that if flushing is disabled,
 * the snapshot method does indeed finishes without waiting for pending records;
 * we set a timeout because the test will not finish if the logic is broken.
 */
@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testDoesNotWaitForPendingRecordsIfFlushingDisabled() throws Throwable {
	final DummyFlinkKafkaProducer<String> producer = new DummyFlinkKafkaProducer<>(
		FakeStandardProducerConfig.get(), new KeyedSerializationSchemaWrapper<>(new SimpleStringSchema()), null);
	producer.setFlushOnCheckpoint(false);

	final KafkaProducer<?, ?> mockProducer = producer.getMockKafkaProducer();

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(producer));

	testHarness.open();

	testHarness.processElement(new StreamRecord<>("msg"));

	// make sure that all callbacks have not been completed
	verify(mockProducer, times(1)).send(any(ProducerRecord.class), any(Callback.class));

	// should return even if there are pending records
	testHarness.snapshot(123L, 123L);

	testHarness.close();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Test
public void testAppendTableSink() throws IOException {
	JDBCAppendTableSink sink = JDBCAppendTableSink.builder()
		.setDrivername("foo")
		.setDBUrl("bar")
		.setQuery("insert into %s (id) values (?)")
		.setParameterTypes(FIELD_TYPES)
		.build();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Row> ds = env.fromCollection(Collections.singleton(Row.of("foo")), ROW_TYPE);
	sink.emitDataStream(ds);

	Collection<Integer> sinkIds = env.getStreamGraph().getSinkIDs();
	assertEquals(1, sinkIds.size());
	int sinkId = sinkIds.iterator().next();

	StreamSink planSink = (StreamSink) env.getStreamGraph().getStreamNode(sinkId).getOperator();
	assertTrue(planSink.getUserFunction() instanceof JDBCSinkFunction);

	JDBCSinkFunction sinkFunction = (JDBCSinkFunction) planSink.getUserFunction();
	assertSame(sink.getOutputFormat(), sinkFunction.outputFormat);
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/**
 * Adds the given sink to this DataStream. Only streams with sinks added
 * will be executed once the {@link StreamExecutionEnvironment#execute()}
 * method is called.
 *
 * @param sinkFunction
 *            The object containing the sink's invoke function.
 * @return The closed DataStream.
 */
public DataStreamSink<T> addSink(SinkFunction<T> sinkFunction) {

	// read the output type of the input Transform to coax out errors about MissingTypeInfo
	transformation.getOutputType();

	// configure the type if needed
	if (sinkFunction instanceof InputTypeConfigurable) {
		((InputTypeConfigurable) sinkFunction).setInputType(getType(), getExecutionConfig());
	}

	StreamSink<T> sinkOperator = new StreamSink<>(clean(sinkFunction));

	DataStreamSink<T> sink = new DataStreamSink<>(this, sinkOperator);

	getExecutionEnvironment().addOperator(sink.getTransformation());
	return sink;
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Override
public <T> StreamSink<T> getProducerSink(
		String topic,
		KeyedSerializationSchema<T> serSchema,
		Properties props,
		FlinkKafkaPartitioner<T> partitioner) {
	FlinkKafkaProducer09<T> prod = new FlinkKafkaProducer09<>(topic, serSchema, props, partitioner);
	prod.setFlushOnCheckpoint(true);
	return new StreamSink<>(prod);
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Test(timeout = 5000)
public void testWaitForPendingUpdatesOnSnapshot() throws Exception {
	TestCassandraSink casSinkFunc = new TestCassandraSink();

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(casSinkFunc));

	testHarness.open();

	CompletableFuture<ResultSet> completableFuture = new CompletableFuture<>();
	ResultSetFuture resultSetFuture = ResultSetFutures.fromCompletableFuture(completableFuture);
	casSinkFunc.setResultFuture(resultSetFuture);

	casSinkFunc.invoke("hello");
	Assert.assertEquals(1, casSinkFunc.getNumOfPendingRecords());

	Thread t = new CheckedThread("Flink-CassandraSinkBaseTest") {
		@Override
		public void go() throws Exception {
			testHarness.snapshot(123L, 123L);
		}
	};
	t.start();
	while (t.getState() != Thread.State.WAITING) {
		Thread.sleep(5);
	}

	Assert.assertEquals(1, casSinkFunc.getNumOfPendingRecords());
	completableFuture.complete(null);
	Assert.assertEquals(0, casSinkFunc.getNumOfPendingRecords());

	testHarness.close();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Test(timeout = 5000)
public void testWaitForPendingUpdatesOnClose() throws Exception {
	TestCassandraSink casSinkFunc = new TestCassandraSink();

	OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(casSinkFunc));

	testHarness.open();

	CompletableFuture<ResultSet> completableFuture = new CompletableFuture<>();
	ResultSetFuture resultSetFuture = ResultSetFutures.fromCompletableFuture(completableFuture);
	casSinkFunc.setResultFuture(resultSetFuture);

	casSinkFunc.invoke("hello");
	Assert.assertEquals(1, casSinkFunc.getNumOfPendingRecords());

	Thread t = new CheckedThread("Flink-CassandraSinkBaseTest") {
		@Override
		public void go() throws Exception {
			testHarness.close();
		}
	};
	t.start();
	while (t.getState() != Thread.State.WAITING) {
		Thread.sleep(5);
	}

	Assert.assertEquals(1, casSinkFunc.getNumOfPendingRecords());
	completableFuture.complete(null);
	Assert.assertEquals(0, casSinkFunc.getNumOfPendingRecords());
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Override
public <T> StreamSink<T> getProducerSink(
		String topic,
		KeyedSerializationSchema<T> serSchema,
		Properties props,
		FlinkKafkaPartitioner<T> partitioner) {
	FlinkKafkaProducer08<T> prod = new FlinkKafkaProducer08<>(
			topic,
			serSchema,
			props,
			partitioner);
	prod.setFlushOnCheckpoint(true);
	return new StreamSink<>(prod);
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
@Override
public <T> StreamSink<T> getProducerSink(String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) {
	return new StreamSink<>(new FlinkKafkaProducer011<>(
		topic,
		serSchema,
		props,
		Optional.ofNullable(partitioner),
		producerSemantic,
		FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE));
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/** Tests that any item failure in the listener callbacks is rethrown on an immediately following invoke call. */
@Test
public void testItemFailureRethrownOnInvoke() throws Throwable {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and its mock item failures
	sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record")));
	testHarness.processElement(new StreamRecord<>("msg"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(ActionRequest.class));

	// manually execute the next bulk request
	sink.manualBulkRequestWithAllPendingRequests();

	try {
		testHarness.processElement(new StreamRecord<>("next msg"));
	} catch (Exception e) {
		// the invoke should have failed with the failure
		Assert.assertTrue(e.getCause().getMessage().contains("artificial failure for record"));

		// test succeeded
		return;
	}

	Assert.fail();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/** Tests that any item failure in the listener callbacks is rethrown on an immediately following checkpoint. */
@Test
public void testItemFailureRethrownOnCheckpoint() throws Throwable {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and its mock item failures
	sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record")));
	testHarness.processElement(new StreamRecord<>("msg"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(ActionRequest.class));

	// manually execute the next bulk request
	sink.manualBulkRequestWithAllPendingRequests();

	try {
		testHarness.snapshot(1L, 1000L);
	} catch (Exception e) {
		// the snapshot should have failed with the failure
		Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for record"));

		// test succeeded
		return;
	}

	Assert.fail();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/** Tests that any bulk failure in the listener callbacks is rethrown on an immediately following invoke call. */
@Test
public void testBulkFailureRethrownOnInvoke() throws Throwable {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and let the whole bulk request fail
	sink.setFailNextBulkRequestCompletely(new Exception("artificial failure for bulk request"));
	testHarness.processElement(new StreamRecord<>("msg"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(ActionRequest.class));

	// manually execute the next bulk request
	sink.manualBulkRequestWithAllPendingRequests();

	try {
		testHarness.processElement(new StreamRecord<>("next msg"));
	} catch (Exception e) {
		// the invoke should have failed with the bulk request failure
		Assert.assertTrue(e.getCause().getMessage().contains("artificial failure for bulk request"));

		// test succeeded
		return;
	}

	Assert.fail();
}
 

import org.apache.flink.streaming.api.operators.StreamSink; //導入依賴的package包/類
/** Tests that any bulk failure in the listener callbacks is rethrown on an immediately following checkpoint. */
@Test
public void testBulkFailureRethrownOnCheckpoint() throws Throwable {
	final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>(
		new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler());

	final OneInputStreamOperatorTestHarness<String, Object> testHarness =
		new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink));

	testHarness.open();

	// setup the next bulk request, and let the whole bulk request fail
	sink.setFailNextBulkRequestCompletely(new Exception("artificial failure for bulk request"));
	testHarness.processElement(new StreamRecord<>("msg"));
	verify(sink.getMockBulkProcessor(), times(1)).add(any(ActionRequest.class));

	// manually execute the next bulk request
	sink.manualBulkRequestWithAllPendingRequests();

	try {
		testHarness.snapshot(1L, 1000L);
	} catch (Exception e) {
		// the snapshot should have failed with the bulk request failure
		Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for bulk request"));

		// test succeeded
		return;
	}

	Assert.fail();
}