Java SourceFunction.SourceContext方法代码示例

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
public LogDataFetcher(SourceFunction.SourceContext<T> sourceContext,
                      RuntimeContext runtimeContext,
                      Properties configProps,
                      LogDeserializationSchema<T> deserializationSchema, LogClientProxy logClient) {
    this.sourceContext = sourceContext;
    this.runtimeContext = runtimeContext;
    this.configProps = configProps;
    this.deserializationSchema = deserializationSchema;
    this.totalNumberOfConsumerSubtasks = runtimeContext.getNumberOfParallelSubtasks();
    this.indexOfThisConsumerSubtask = runtimeContext.getIndexOfThisSubtask();
    this.checkpointLock = sourceContext.getCheckpointLock();
    this.subscribedShardsState = new LinkedList<LogstoreShardState>();
    this.shardConsumersExecutor = createShardConsumersThreadPool(runtimeContext.getTaskNameWithSubtasks());
    this.error = new AtomicReference<Throwable>();
    this.logProject = configProps.getProperty(ConfigConstants.LOG_PROJECT);
    this.logStore = configProps.getProperty(ConfigConstants.LOG_LOGSTORE);
    this.logClient = logClient;
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Override
public void run(SourceFunction.SourceContext<model.LocalWeatherData> sourceContext) throws Exception {

    // The Source needs to be Serializable, so we have to construct the Paths at this point:
    final Path csvStationPath = FileSystems.getDefault().getPath(stationFilePath);
    final Path csvLocalWeatherDataPath = FileSystems.getDefault().getPath(localWeatherDataFilePath);

    // Get the Stream of LocalWeatherData Elements in the CSV File:
    try(Stream<model.LocalWeatherData> stream = getLocalWeatherData(csvStationPath, csvLocalWeatherDataPath)) {

        // We need to get an iterator, since the SourceFunction has to break out of its main loop on cancellation:
        Iterator<model.LocalWeatherData> iterator = stream.iterator();

        // Make sure to cancel, when the Source function is canceled by an external event:
        while (isRunning && iterator.hasNext()) {
            sourceContext.collect(iterator.next());
        }
    }
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Before
public void beforeTest() throws Exception {
  feederActorSystem = ActorSystem.create("feederActorSystem",
    getFeederActorConfig());

  sourceContext = new DummySourceContext();

  sourceThread = new Thread(new Runnable() {
    @Override
    public void run() {
      try {
        SourceFunction.SourceContext<Object> sourceContext =
          new DummySourceContext();
        source.run(sourceContext);
      } catch (Exception e) {
        exception = e;
      }
    }
  });
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
/**
 * Creates a Kinesis Data Fetcher.
 *
 * @param streams the streams to subscribe to
 * @param sourceContext context of the source function
 * @param runtimeContext this subtask's runtime context
 * @param configProps the consumer configuration properties
 * @param deserializationSchema deserialization schema
 */
public KinesisDataFetcher(List<String> streams,
						SourceFunction.SourceContext<T> sourceContext,
						RuntimeContext runtimeContext,
						Properties configProps,
						KinesisDeserializationSchema<T> deserializationSchema) {
	this(streams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		runtimeContext,
		configProps,
		deserializationSchema,
		new AtomicReference<Throwable>(),
		new LinkedList<KinesisStreamShardState>(),
		createInitialSubscribedStreamsToLastDiscoveredShardsState(streams),
		KinesisProxy.create(configProps));
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Override
public void run(SourceFunction.SourceContext<TaxiRide> sourceContext) throws Exception {

	gzipStream = new GZIPInputStream(new FileInputStream(dataFilePath));
	reader = new BufferedReader(new InputStreamReader(gzipStream, "UTF-8"));

	generateUnorderedStream(sourceContext);

	this.reader.close();
	this.reader = null;
	this.gzipStream.close();
	this.gzipStream = null;

}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Override
public void run(SourceFunction.SourceContext<Object> ctx) throws Exception {
  LOG.info("Starting the Receiver actor {}", actorName);
  receiverActor = receiverActorSystem.actorOf(
    Props.create(classForActor, ctx, urlOfPublisher, autoAck), actorName);

  LOG.info("Started the Receiver actor {} successfully", actorName);
  receiverActorSystem.awaitTermination();
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
/** This method is exposed for tests that need to mock the KinesisDataFetcher in the consumer. */
protected KinesisDataFetcher<T> createFetcher(
		List<String> streams,
		SourceFunction.SourceContext<T> sourceContext,
		RuntimeContext runtimeContext,
		Properties configProps,
		KinesisDeserializationSchema<T> deserializationSchema) {

	return new KinesisDataFetcher<>(streams, sourceContext, runtimeContext, configProps, deserializationSchema);
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Override
protected KinesisDataFetcher<T> createFetcher(
		List<String> streams,
		SourceFunction.SourceContext<T> sourceContext,
		RuntimeContext runtimeContext,
		Properties configProps,
		KinesisDeserializationSchema<T> deserializationSchema) {
	return fetcher;
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
/**
 * Depending on the {@link TimeCharacteristic}, this method will return the adequate
 * {@link org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext}. That is:
 * <ul>
 *     <li>{@link TimeCharacteristic#IngestionTime} = {@code AutomaticWatermarkContext}</li>
 *     <li>{@link TimeCharacteristic#ProcessingTime} = {@code NonTimestampContext}</li>
 *     <li>{@link TimeCharacteristic#EventTime} = {@code ManualWatermarkContext}</li>
 * </ul>
 * */
public static <OUT> SourceFunction.SourceContext<OUT> getSourceContext(
		TimeCharacteristic timeCharacteristic,
		ProcessingTimeService processingTimeService,
		Object checkpointLock,
		StreamStatusMaintainer streamStatusMaintainer,
		Output<StreamRecord<OUT>> output,
		long watermarkInterval,
		long idleTimeout) {

	final SourceFunction.SourceContext<OUT> ctx;
	switch (timeCharacteristic) {
		case EventTime:
			ctx = new ManualWatermarkContext<>(
				output,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case IngestionTime:
			ctx = new AutomaticWatermarkContext<>(
				output,
				watermarkInterval,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case ProcessingTime:
			ctx = new NonTimestampContext<>(checkpointLock, output);
			break;
		default:
			throw new IllegalArgumentException(String.valueOf(timeCharacteristic));
	}
	return ctx;
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
private static <T extends Serializable> List<T> runNonRichSourceFunction(SourceFunction<T> sourceFunction) {
	final List<T> outputs = new ArrayList<>();
	try {
		SourceFunction.SourceContext<T> ctx = new CollectingSourceContext<T>(new Object(), outputs);
		sourceFunction.run(ctx);
	} catch (Exception e) {
		throw new RuntimeException("Cannot invoke source.", e);
	}
	return outputs;
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Override
public void run(SourceFunction.SourceContext<Tuple2<Integer, String>> ctx) throws Exception {
	for (int i = 0; i < NUM_ELEMENTS && running; i++) {
		ctx.collect(Tuple2.of(i, DATA_PREFIX + i));
	}
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private static SourceFunction.SourceContext<String> getMockedSourceContext() {
	return Mockito.mock(SourceFunction.SourceContext.class);
}
 

import org.apache.flink.streaming.api.functions.source.SourceFunction; //导入方法依赖的package包/类
@Test
public void testCheckpointAndRestore() {
	try {
		final int numElements = 10000;

		List<Integer> data = new ArrayList<Integer>(numElements);
		List<Integer> result = new ArrayList<Integer>(numElements);

		for (int i = 0; i < numElements; i++) {
			data.add(i);
		}

		final FromElementsFunction<Integer> source = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
		StreamSource<Integer, FromElementsFunction<Integer>> src = new StreamSource<>(source);
		AbstractStreamOperatorTestHarness<Integer> testHarness =
			new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
		testHarness.open();

		final SourceFunction.SourceContext<Integer> ctx = new ListSourceContext<Integer>(result, 2L);

		final Throwable[] error = new Throwable[1];

		// run the source asynchronously
		Thread runner = new Thread() {
			@Override
			public void run() {
				try {
					source.run(ctx);
				}
				catch (Throwable t) {
					error[0] = t;
				}
			}
		};
		runner.start();

		// wait for a bit
		Thread.sleep(1000);

		// make a checkpoint
		List<Integer> checkpointData = new ArrayList<>(numElements);
		OperatorStateHandles handles = null;
		synchronized (ctx.getCheckpointLock()) {
			handles = testHarness.snapshot(566, System.currentTimeMillis());
			checkpointData.addAll(result);
		}

		// cancel the source
		source.cancel();
		runner.join();

		// check for errors
		if (error[0] != null) {
			System.err.println("Error in asynchronous source runner");
			error[0].printStackTrace();
			fail("Error in asynchronous source runner");
		}

		final FromElementsFunction<Integer> sourceCopy = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
		StreamSource<Integer, FromElementsFunction<Integer>> srcCopy = new StreamSource<>(sourceCopy);
		AbstractStreamOperatorTestHarness<Integer> testHarnessCopy =
			new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
		testHarnessCopy.setup();
		testHarnessCopy.initializeState(handles);
		testHarnessCopy.open();

		// recovery run
		SourceFunction.SourceContext<Integer> newCtx = new ListSourceContext<>(checkpointData);

		sourceCopy.run(newCtx);

		assertEquals(data, checkpointData);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}