Java StreamStatusMaintainer類代碼示例

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
private AutomaticWatermarkContext(
		final Output<StreamRecord<T>> output,
		final long watermarkInterval,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");

	Preconditions.checkArgument(watermarkInterval >= 1L, "The watermark interval cannot be smaller than 1 ms.");
	this.watermarkInterval = watermarkInterval;

	this.reuse = new StreamRecord<>(null);

	this.lastRecordTime = Long.MIN_VALUE;

	long now = this.timeService.getCurrentProcessingTime();
	this.nextWatermarkTimer = this.timeService.registerTimer(now + watermarkInterval,
		new WatermarkEmittingTask(this.timeService, checkpointLock, output));
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
/**
 * Create a watermark context.
 *
 * @param timeService the time service to schedule idleness detection tasks
 * @param checkpointLock the checkpoint lock
 * @param streamStatusMaintainer the stream status maintainer to toggle and retrieve current status
 * @param idleTimeout (-1 if idleness checking is disabled)
 */
public WatermarkContext(
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	this.timeService = Preconditions.checkNotNull(timeService, "Time Service cannot be null.");
	this.checkpointLock = Preconditions.checkNotNull(checkpointLock, "Checkpoint Lock cannot be null.");
	this.streamStatusMaintainer = Preconditions.checkNotNull(streamStatusMaintainer, "Stream Status Maintainer cannot be null.");

	if (idleTimeout != -1) {
		Preconditions.checkArgument(idleTimeout >= 1, "The idle timeout cannot be smaller than 1 ms.");
	}
	this.idleTimeout = idleTimeout;

	scheduleNextIdleDetectionTask();
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@Test
public void testNoMaxWatermarkOnImmediateCancel() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StreamSource<String, InfiniteSource<String>> operator =
			new StreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, 0);
	operator.cancel();

	// run and exit
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@Test
public void testNoMaxWatermarkOnImmediateStop() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
			new StoppableStreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, 0);
	operator.stop();

	// run and stop
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
/**
 * Depending on the {@link TimeCharacteristic}, this method will return the adequate
 * {@link org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext}. That is:
 * <ul>
 *     <li>{@link TimeCharacteristic#IngestionTime} = {@code AutomaticWatermarkContext}</li>
 *     <li>{@link TimeCharacteristic#ProcessingTime} = {@code NonTimestampContext}</li>
 *     <li>{@link TimeCharacteristic#EventTime} = {@code ManualWatermarkContext}</li>
 * </ul>
 * */
public static <OUT> SourceFunction.SourceContext<OUT> getSourceContext(
		TimeCharacteristic timeCharacteristic,
		ProcessingTimeService processingTimeService,
		Object checkpointLock,
		StreamStatusMaintainer streamStatusMaintainer,
		Output<StreamRecord<OUT>> output,
		long watermarkInterval,
		long idleTimeout) {

	final SourceFunction.SourceContext<OUT> ctx;
	switch (timeCharacteristic) {
		case EventTime:
			ctx = new ManualWatermarkContext<>(
				output,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case IngestionTime:
			ctx = new AutomaticWatermarkContext<>(
				output,
				watermarkInterval,
				processingTimeService,
				checkpointLock,
				streamStatusMaintainer,
				idleTimeout);

			break;
		case ProcessingTime:
			ctx = new NonTimestampContext<>(checkpointLock, output);
			break;
		default:
			throw new IllegalArgumentException(String.valueOf(timeCharacteristic));
	}
	return ctx;
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
private ManualWatermarkContext(
		final Output<StreamRecord<T>> output,
		final ProcessingTimeService timeService,
		final Object checkpointLock,
		final StreamStatusMaintainer streamStatusMaintainer,
		final long idleTimeout) {

	super(timeService, checkpointLock, streamStatusMaintainer, idleTimeout);

	this.output = Preconditions.checkNotNull(output, "The output cannot be null.");
	this.reuse = new StreamRecord<>(null);
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<OUT>> collector) throws Exception {
	ACTUAL_ORDER_TRACKING.add("OPERATOR::run");
	super.run(lockingObject, streamStatusMaintainer, collector);
	runStarted.trigger();
	runFinish.await();
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@Override
public void run(Object lockingObject,
				StreamStatusMaintainer streamStatusMaintainer,
				Output<StreamRecord<Long>> collector) throws Exception {
	while (!canceled) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException ignored) {}
	}
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@Test
public void testEmitMaxWatermarkForFiniteSource() throws Exception {

	// regular stream source operator
	StreamSource<String, FiniteSource<String>> operator =
			new StreamSource<>(new FiniteSource<String>());

	final List<StreamElement> output = new ArrayList<>();

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, 0);
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertEquals(1, output.size());
	assertEquals(Watermark.MAX_WATERMARK, output.get(0));
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@Test
public void testNoMaxWatermarkOnAsyncStop() throws Exception {

	final List<StreamElement> output = new ArrayList<>();

	// regular stream source operator
	final StoppableStreamSource<String, InfiniteSource<String>> operator =
			new StoppableStreamSource<>(new InfiniteSource<String>());

	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, 0);

	// trigger an async cancel in a bit
	new Thread("canceler") {
		@Override
		public void run() {
			try {
				Thread.sleep(200);
			} catch (InterruptedException ignored) {}
			operator.stop();
		}
	}.start();

	// run and wait to be stopped
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<String>(output));

	assertTrue(output.isEmpty());
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
public void run(final Object lockingObject, final StreamStatusMaintainer streamStatusMaintainer) throws Exception {
	run(lockingObject, streamStatusMaintainer, output);
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
public StreamStatusMaintainer getStreamStatusMaintainer() {
	return operatorChain;
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@SuppressWarnings("unchecked")
public StreamInputProcessor(
		InputGate[] inputGates,
		TypeSerializer<IN> inputSerializer,
		StreamTask<?, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		OneInputStreamOperator<IN, ?> streamOperator) throws IOException {

	InputGate inputGate = InputGateUtil.createInputGate(inputGates);

	if (checkpointMode == CheckpointingMode.EXACTLY_ONCE) {
		long maxAlign = taskManagerConfig.getLong(TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT);
		if (!(maxAlign == -1 || maxAlign > 0)) {
			throw new IllegalConfigurationException(
					TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT.key()
					+ " must be positive or -1 (infinite)");
		}
		this.barrierHandler = new BarrierBuffer(inputGate, ioManager, maxAlign);
	}
	else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
		this.barrierHandler = new BarrierTracker(inputGate);
	}
	else {
		throw new IllegalArgumentException("Unrecognized Checkpointing Mode: " + checkpointMode);
	}

	if (checkpointedTask != null) {
		this.barrierHandler.registerCheckpointEventHandler(checkpointedTask);
	}

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN> ser = new StreamElementSerializer<>(inputSerializer);
	this.deserializationDelegate = new NonReusingDeserializationDelegate<>(ser);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
				ioManager.getSpillingDirectoriesPaths());
	}

	this.numInputChannels = inputGate.getNumberOfInputChannels();

	this.lastEmittedWatermark = Long.MIN_VALUE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve = new StatusWatermarkValve(
			numInputChannels,
			new ForwardingValveOutputHandler(streamOperator, lock));
}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
@SuppressWarnings("unchecked")
public StreamTwoInputProcessor(
		Collection<InputGate> inputGates1,
		Collection<InputGate> inputGates2,
		TypeSerializer<IN1> inputSerializer1,
		TypeSerializer<IN2> inputSerializer2,
		TwoInputStreamTask<IN1, IN2, ?> checkpointedTask,
		CheckpointingMode checkpointMode,
		Object lock,
		IOManager ioManager,
		Configuration taskManagerConfig,
		StreamStatusMaintainer streamStatusMaintainer,
		TwoInputStreamOperator<IN1, IN2, ?> streamOperator) throws IOException {

	final InputGate inputGate = InputGateUtil.createInputGate(inputGates1, inputGates2);

	if (checkpointMode == CheckpointingMode.EXACTLY_ONCE) {
		long maxAlign = taskManagerConfig.getLong(TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT);
		if (!(maxAlign == -1 || maxAlign > 0)) {
			throw new IllegalConfigurationException(
					TaskManagerOptions.TASK_CHECKPOINT_ALIGNMENT_BYTES_LIMIT.key()
							+ " must be positive or -1 (infinite)");
		}
		this.barrierHandler = new BarrierBuffer(inputGate, ioManager, maxAlign);
	}
	else if (checkpointMode == CheckpointingMode.AT_LEAST_ONCE) {
		this.barrierHandler = new BarrierTracker(inputGate);
	}
	else {
		throw new IllegalArgumentException("Unrecognized CheckpointingMode: " + checkpointMode);
	}

	if (checkpointedTask != null) {
		this.barrierHandler.registerCheckpointEventHandler(checkpointedTask);
	}

	this.lock = checkNotNull(lock);

	StreamElementSerializer<IN1> ser1 = new StreamElementSerializer<>(inputSerializer1);
	this.deserializationDelegate1 = new NonReusingDeserializationDelegate<>(ser1);

	StreamElementSerializer<IN2> ser2 = new StreamElementSerializer<>(inputSerializer2);
	this.deserializationDelegate2 = new NonReusingDeserializationDelegate<>(ser2);

	// Initialize one deserializer per input channel
	this.recordDeserializers = new SpillingAdaptiveSpanningRecordDeserializer[inputGate.getNumberOfInputChannels()];

	for (int i = 0; i < recordDeserializers.length; i++) {
		recordDeserializers[i] = new SpillingAdaptiveSpanningRecordDeserializer<>(
				ioManager.getSpillingDirectoriesPaths());
	}

	// determine which unioned channels belong to input 1 and which belong to input 2
	int numInputChannels1 = 0;
	for (InputGate gate: inputGates1) {
		numInputChannels1 += gate.getNumberOfInputChannels();
	}

	this.numInputChannels1 = numInputChannels1;
	this.numInputChannels2 = inputGate.getNumberOfInputChannels() - numInputChannels1;

	this.lastEmittedWatermark1 = Long.MIN_VALUE;
	this.lastEmittedWatermark2 = Long.MIN_VALUE;

	this.firstStatus = StreamStatus.ACTIVE;
	this.secondStatus = StreamStatus.ACTIVE;

	this.streamStatusMaintainer = checkNotNull(streamStatusMaintainer);
	this.streamOperator = checkNotNull(streamOperator);

	this.statusWatermarkValve1 = new StatusWatermarkValve(numInputChannels1, new ForwardingValveOutputHandler1(streamOperator, lock));
	this.statusWatermarkValve2 = new StatusWatermarkValve(numInputChannels2, new ForwardingValveOutputHandler2(streamOperator, lock));

}
 

import org.apache.flink.streaming.runtime.streamstatus.StreamStatusMaintainer; //導入依賴的package包/類
/**
 * Test that latency marks are emitted.
 */
@Test
public void testLatencyMarkEmission() throws Exception {
	final List<StreamElement> output = new ArrayList<>();

	final long maxProcessingTime = 100L;
	final long latencyMarkInterval = 10L;

	final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
	testProcessingTimeService.setCurrentTime(0L);
	final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);

	// regular stream source operator
	final StreamSource<Long, ProcessingTimeServiceSource> operator =
			new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));

	// emit latency marks every 10 milliseconds.
	setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, latencyMarkInterval, testProcessingTimeService);

	// run and wait to be stopped
	operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output));

	int numberLatencyMarkers = (int) (maxProcessingTime / latencyMarkInterval) + 1;

	assertEquals(
		numberLatencyMarkers + 1, // + 1 is the final watermark element
		output.size());

	long timestamp = 0L;

	int i = 0;
	// and that its only latency markers + a final watermark
	for (; i < output.size() - 1; i++) {
		StreamElement se = output.get(i);
		Assert.assertTrue(se.isLatencyMarker());
		Assert.assertEquals(-1, se.asLatencyMarker().getVertexID());
		Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());
		Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp);

		timestamp += latencyMarkInterval;
	}

	Assert.assertTrue(output.get(i).isWatermark());
}