Java StreamRecord类代码示例

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Override
public void processElement(StreamRecord<T> element) throws Exception {
    if (!element.hasTimestamp()) {
        // elements with no time component are simply forwarded.
        // likely cause: the time characteristic of the program is not event-time.
        output.collect(element);
        return;
    }

    // In event-time processing we assume correctness of the watermark.
    // Events with timestamp smaller than (or equal to) the last seen watermark are considered late.
    // FUTURE: emit late elements to a side output

    if (element.getTimestamp() > lastWatermark) {
        // we have an event with a valid timestamp, so
        // we buffer it until we receive the proper watermark.
        saveRegisterWatermarkTimer();
        bufferEvent(element);
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
/**
 * Buffers an element for future processing.
 *
 * @param element the element to buffer.
 * @throws Exception if any error occurs.
 */
private void bufferEvent(StreamRecord<T> element) throws Exception {

    assert element.hasTimestamp();
    long timestamp = element.getTimestamp();

    List<T> elementsForTimestamp = elementQueueState.get(timestamp);
    if (elementsForTimestamp == null) {
        elementsForTimestamp = new ArrayList<>(1);
    }

    if (getRuntimeContext().getExecutionConfig().isObjectReuseEnabled()) {
        // copy the object so that the original object may be reused
        elementsForTimestamp.add(inputSerializer.copy(element.getValue()));
    } else {
        elementsForTimestamp.add(element.getValue());
    }
    elementQueueState.put(timestamp, elementsForTimestamp);
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
/**
 * Occurs when an event-time timer fires due to watermark progression.
 *
 * @param timer the timer details.
 */
@Override
public void onEventTime(InternalTimer<K, VoidNamespace> timer) throws Exception {

    long currentWatermark = internalTimerService.currentWatermark();

    PriorityQueue<Long> sortedTimestamps = getSortedTimestamps();
    while (!sortedTimestamps.isEmpty() && sortedTimestamps.peek() <= currentWatermark) {
        long timestamp = sortedTimestamps.poll();
        for (T event : elementQueueState.get(timestamp)) {
            output.collect(new StreamRecord<>(event, timestamp));
        }
        elementQueueState.remove(timestamp);
    }

    if (sortedTimestamps.isEmpty()) {
        elementQueueState.clear();
    }

    if (!sortedTimestamps.isEmpty()) {
        saveRegisterWatermarkTimer();
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Override
public void receive(Event[] events) {
	for (Event event : events) {
		if (typeInfo == null || Map.class.isAssignableFrom(typeInfo.getTypeClass())) {
			collectedRecords.add(new StreamRecord<R>((R) toMap(event), event.getTimestamp()));
		} else if (typeInfo.isTupleType()) {
			Tuple tuple = this.toTuple(event);
			collectedRecords.add(new StreamRecord<R>((R) tuple, event.getTimestamp()));
		} else if (typeInfo instanceof PojoTypeInfo) {
			R obj;
			try {
				obj = objectMapper.convertValue(toMap(event), typeInfo.getTypeClass());
			} catch (IllegalArgumentException ex) {
				LOGGER.error("Failed to map event: " + event + " into type: " + typeInfo, ex);
				throw ex;
			}
			collectedRecords.add(new StreamRecord<R>(obj, event.getTimestamp()));
		} else {
			throw new IllegalArgumentException("Unable to format " + event + " as type " + typeInfo);
		}
	}
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Override
public int compare(Object o1, Object o2) {
	if (o1 instanceof Watermark || o2 instanceof Watermark) {
		return 0;
	} else {
		StreamRecord<Tuple3<String, Long, Long>> sr0 = (StreamRecord<Tuple3<String, Long, Long>>) o1;
		StreamRecord<Tuple3<String, Long, Long>> sr1 = (StreamRecord<Tuple3<String, Long, Long>>) o2;
		if (sr0.getTimestamp() != sr1.getTimestamp()) {
			return (int) (sr0.getTimestamp() - sr1.getTimestamp());
		}
		int comparison = sr0.getValue().f0.compareTo(sr1.getValue().f0);
		if (comparison != 0) {
			return comparison;
		} else {
			comparison = (int) (sr0.getValue().f1 - sr1.getValue().f1);
			if (comparison != 0) {
				return comparison;
			}
			return (int) (sr0.getValue().f1 - sr1.getValue().f1);
		}
	}
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
protected void processInput(Network network, IN record, long timestamp) throws Exception {

        if(userFunction.reset(record)) {
            network.reset();
            LOG.debug("network reset");
        }

        Object input = convertToInput(record, timestamp);
        Inference inference = network.computeImmediate(input);

        if(inference != null) {
            NetworkInference outputInference = NetworkInference.fromInference(inference);
            StreamRecord<Tuple2<IN,NetworkInference>> streamRecord = new StreamRecord<>(
                    new Tuple2<>(record, outputInference),
                    timestamp);
            output.collect(streamRecord);
        }
    }
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Test
public void testMap() throws Exception {
	StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));

	expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
private EPServiceProvider getServiceProvider(String context) throws IOException {
    EPServiceProvider serviceProvider = engineState.value();
    if (serviceProvider != null) {
        return serviceProvider;
    }
    synchronized (lock) {
        serviceProvider = engineState.value();
        if (serviceProvider == null) {
            Configuration configuration = new Configuration();
            configuration.getEngineDefaults().getThreading().setInternalTimerEnabled(false);
            serviceProvider = EPServiceProviderManager.getProvider(context, configuration);
            serviceProvider.getEPAdministrator().getConfiguration().addEventType(inputType.getTypeClass());
            serviceProvider.getEPRuntime().sendEvent(new CurrentTimeEvent(0));
            EPStatement statement = query.createStatement(serviceProvider.getEPAdministrator());

            statement.addListener((newData, oldData) -> {
                for (EventBean event : newData) {
                    EsperSelectFunction<OUT> userFunction = getUserFunction();
                    try {
                        output.collect(new StreamRecord<>((userFunction.select(event))));
                    } catch (Exception e) {
                        throw new RuntimeException(e);
                    }
                }
            });
            this.engineState.update(serviceProvider);
            return serviceProvider;

        } else {
            return engineState.value();
        }
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Test
public void testProcessingTime() throws Exception {
    testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.processElement(new StreamRecord<>(new Tuple2<>(K1, 0L)));
    Queue<Object> actual = testHarness.getOutput();
    Queue<Object> expected = new ConcurrentLinkedQueue<>();
    expected.add(new StreamRecord<>(new Tuple2<>(K1, 0L)));
    TestHarnessUtil.assertOutputEquals("Unexpected output", expected, actual);
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
/**
 * Tests the {@code processElement} method.
 * See also: {@code testNonTransactionalWriterProcessElementAccounting}, {@code testNonTransactionalWriterProcessElementErrorHandling}
 */
@Test
public void testNonTransactionalWriterProcessElementWrite() throws Exception {
    try (NonTransactionalWriterTestContext context = new NonTransactionalWriterTestContext(PravegaWriterMode.ATLEAST_ONCE)) {
        try (StreamSinkOperatorTestHarness<Integer> testHarness = createTestHarness(context.sinkFunction)) {
            testHarness.open();
            CompletableFuture<Void> e1Future = context.prepareWrite();
            StreamRecord<Integer> e1 = new StreamRecord<>(1, 1L);
            testHarness.processElement(e1);
            verify(context.pravegaWriter).writeEvent(ROUTING_KEY, e1.getValue());
            e1Future.complete(null);
        }
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
/**
 * Tests the accounting of pending writes.
 */
@Test
public void testNonTransactionalWriterProcessElementAccounting() throws Exception {
    try (NonTransactionalWriterTestContext context = new NonTransactionalWriterTestContext(PravegaWriterMode.ATLEAST_ONCE)) {
        try (StreamSinkOperatorTestHarness<Integer> testHarness = createTestHarness(context.sinkFunction)) {
            testHarness.open();
            FlinkPravegaWriter.NonTransactionalWriter internalWriter = (FlinkPravegaWriter.NonTransactionalWriter) context.sinkFunction.writer;

            CompletableFuture<Void> e1Future = context.prepareWrite();
            StreamRecord<Integer> e1 = new StreamRecord<>(1, 1L);
            testHarness.processElement(e1);
            Assert.assertEquals(1, internalWriter.pendingWritesCount.get());

            CompletableFuture<Void> e2Future = context.prepareWrite();
            StreamRecord<Integer> e2 = new StreamRecord<>(2, 2L);
            testHarness.processElement(e2);
            Assert.assertEquals(2, internalWriter.pendingWritesCount.get());

            CompletableFuture<Void> e3Future = context.prepareWrite();
            StreamRecord<Integer> e3 = new StreamRecord<>(3, 3L);
            testHarness.processElement(e3);
            Assert.assertEquals(3, internalWriter.pendingWritesCount.get());

            e1Future.complete(null);
            e2Future.completeExceptionally(new IntentionalRuntimeException());
            e3Future.complete(null);
            Assert.assertEquals(0, internalWriter.pendingWritesCount.get());

            // clear the error for test simplicity
            internalWriter.writeError.set(null);
        }
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
/**
 * Tests the handling of write errors.
 */
@Test
public void testNonTransactionalWriterProcessElementErrorHandling() throws Exception {
    try (NonTransactionalWriterTestContext context = new NonTransactionalWriterTestContext(PravegaWriterMode.ATLEAST_ONCE)) {
        try (StreamSinkOperatorTestHarness<Integer> testHarness = createTestHarness(context.sinkFunction)) {
            testHarness.open();
            FlinkPravegaWriter.NonTransactionalWriter internalWriter = (FlinkPravegaWriter.NonTransactionalWriter) context.sinkFunction.writer;

            CompletableFuture<Void> e1Future = context.prepareWrite();
            StreamRecord<Integer> e1 = new StreamRecord<>(1, 1L);
            testHarness.processElement(e1);
            e1Future.completeExceptionally(new IntentionalRuntimeException());
            Assert.assertNotNull(internalWriter.writeError.get());

            StreamRecord<Integer> e2 = new StreamRecord<>(2, 2L);
            try {
                testHarness.processElement(e2);
                Assert.fail("expected an IOException due to a prior write error");
            } catch (IOException e) {
                // expected
            }

            // clear the error for test simplicity
            internalWriter.writeError.set(null);
        }
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
/**
 * Tests the {@code processElement} method.
 */
@Test
public void testTransactionalWriterProcessElementWrite() throws Exception {
    try (TransactionalWriterTestContext context = new TransactionalWriterTestContext(PravegaWriterMode.EXACTLY_ONCE)) {
        Transaction<Integer> trans = context.prepareTransaction();
        try (StreamSinkOperatorTestHarness<Integer> testHarness = createTestHarness(context.sinkFunction)) {
            testHarness.open();
            StreamRecord<Integer> e1 = new StreamRecord<>(1, 1L);
            testHarness.processElement(e1);
            verify(trans).writeEvent(ROUTING_KEY, e1.getValue());
        }
    }
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Override
public int compare(StreamRecord<IN> o1, StreamRecord<IN> o2) {
	if (o1.getTimestamp() < o2.getTimestamp()) {
		return -1;
	} else if (o1.getTimestamp() > o2.getTimestamp()) {
		return 1;
	} else {
		return 0;
	}
}
 

import org.apache.flink.streaming.runtime.streamrecord.StreamRecord; //导入依赖的package包/类
@Override
protected PriorityQueue<StreamRecord<Tuple2<String, IN>>> restoreQueuerState(DataInputView dataInputView) throws IOException {
	int sizeOfQueue = dataInputView.readInt();
	PriorityQueue<StreamRecord<Tuple2<String, IN>>> priorityQueue = new PriorityQueue<>(sizeOfQueue);
	for (int i = 0; i < sizeOfQueue; i++) {
		String streamId = dataInputView.readUTF();
		StreamElement streamElement = getStreamRecordSerializer(streamId).deserialize(dataInputView);
		priorityQueue.offer(streamElement.<Tuple2<String, IN>>asRecord());
	}
	return priorityQueue;
}