本文整理汇总了Java中org.apache.flink.runtime.state.FunctionInitializationContext.isRestored方法的典型用法代码示例。如果您正苦于以下问题:Java FunctionInitializationContext.isRestored方法的具体用法?Java FunctionInitializationContext.isRestored怎么用?Java FunctionInitializationContext.isRestored使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.flink.runtime.state.FunctionInitializationContext
的用法示例。
在下文中一共展示了FunctionInitializationContext.isRestored方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
if (checkpointCoder == null) {
// no checkpoint coder available in this source
return;
}
OperatorStateStore stateStore = context.getOperatorStateStore();
CoderTypeInformation<
KV<? extends UnboundedSource<OutputT, CheckpointMarkT>, CheckpointMarkT>>
typeInformation = (CoderTypeInformation) new CoderTypeInformation<>(checkpointCoder);
stateForCheckpoint = stateStore.getOperatorState(
new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME,
typeInformation.createSerializer(new ExecutionConfig())));
if (context.isRestored()) {
isRestored = true;
LOG.info("Having restore state in the UnbounedSourceWrapper.");
} else {
LOG.info("No restore state for UnbounedSourceWrapper.");
}
}
示例2: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
TypeInformation<Tuple2<StreamShardMetadata, SequenceNumber>> shardsStateTypeInfo = new TupleTypeInfo<>(
TypeInformation.of(StreamShardMetadata.class),
TypeInformation.of(SequenceNumber.class));
sequenceNumsStateForCheckpoint = context.getOperatorStateStore().getUnionListState(
new ListStateDescriptor<>(sequenceNumsStateStoreName, shardsStateTypeInfo));
if (context.isRestored()) {
if (sequenceNumsToRestore == null) {
sequenceNumsToRestore = new HashMap<>();
for (Tuple2<StreamShardMetadata, SequenceNumber> kinesisSequenceNumber : sequenceNumsStateForCheckpoint.get()) {
sequenceNumsToRestore.put(kinesisSequenceNumber.f0, kinesisSequenceNumber.f1);
}
LOG.info("Setting restore state in the FlinkKinesisConsumer. Using the following offsets: {}",
sequenceNumsToRestore);
}
} else {
LOG.info("No restore state for FlinkKinesisConsumer.");
}
}
示例3: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
if (broadcast) {
this.counterPartitions = context
.getOperatorStateStore()
.getUnionListState(new ListStateDescriptor<>("counter_partitions", IntSerializer.INSTANCE));
} else {
this.counterPartitions = context
.getOperatorStateStore()
.getListState(new ListStateDescriptor<>("counter_partitions", IntSerializer.INSTANCE));
}
if (context.isRestored()) {
for (int v : counterPartitions.get()) {
counter += v;
}
checkCorrectRestore[getRuntimeContext().getIndexOfThisSubtask()] = counter;
}
}
示例4: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkState(this.checkpointedState == null,
"The " + getClass().getSimpleName() + " has already been initialized.");
this.checkpointedState = context.getOperatorStateStore().getListState(
new ListStateDescriptor<>(
"from-elements-state",
IntSerializer.INSTANCE
)
);
if (context.isRestored()) {
List<Integer> retrievedStates = new ArrayList<>();
for (Integer entry : this.checkpointedState.get()) {
retrievedStates.add(entry);
}
// given that the parallelism of the function is 1, we can only have 1 state
Preconditions.checkArgument(retrievedStates.size() == 1,
getClass().getSimpleName() + " retrieved invalid state.");
this.numElementsToSkip = retrievedStates.get(0);
}
}
示例5: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
OperatorStateStore stateStore = context.getOperatorStateStore();
offsetsStateForCheckpoint = stateStore.getSerializableListState(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME);
if (context.isRestored()) {
restoreToOffset = new HashMap<>();
for (Tuple2<KafkaTopicPartition, Long> kafkaOffset : offsetsStateForCheckpoint.get()) {
restoreToOffset.put(kafkaOffset.f0, kafkaOffset.f1);
}
LOG.info("Setting restore state in the FlinkKafkaConsumer.");
if (LOG.isDebugEnabled()) {
LOG.debug("Using the following offsets: {}", restoreToOffset);
}
} else {
LOG.info("No restore state for FlinkKafkaConsumer.");
}
}
示例6: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
public void initializeState(FunctionInitializationContext context) throws Exception {
LOG.info("initializeState...");
TypeInformation<Tuple2<LogstoreShardMeta, String>> shardsStateTypeInfo = new TupleTypeInfo<Tuple2<LogstoreShardMeta, String>>(
TypeInformation.of(LogstoreShardMeta.class),
TypeInformation.of(String.class));
cursorStateForCheckpoint = context.getOperatorStateStore().getUnionListState(
new ListStateDescriptor(curcorStateStoreName, shardsStateTypeInfo));
if (context.isRestored()) {
if (cursorsToRestore == null) {
cursorsToRestore = new HashMap<LogstoreShardMeta, String>();
for (Tuple2<LogstoreShardMeta, String> cursor : cursorStateForCheckpoint.get()) {
LOG.info("initializeState, project: {}, logstore: {}, shard: {}, checkpoint: {}", logProject, logStore, cursor.f0.toString(), cursor.f1);
cursorsToRestore.put(cursor.f0, cursor.f1);
if (consumerGroupName != null && logClient != null) {
logClient.updateCheckpoint(logProject, logStore, consumerGroupName, "flinkTask-" + getRuntimeContext().getIndexOfThisSubtask() + "Of" + getRuntimeContext().getNumberOfParallelSubtasks(), cursor.f0.getShardId(), cursor.f1);
}
}
LOG.info("Setting restore state in the FlinkLogConsumer. Using the following offsets: {}",
cursorsToRestore);
}
}
else {
LOG.info("No restore state for FlinkLogConsumer.");
}
}
示例7: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkArgument(this.restoredBucketStates == null, "The operator has already been initialized.");
try {
initFileSystem();
} catch (IOException e) {
LOG.error("Error while creating FileSystem when initializing the state of the TODBucketingSink.", e);
throw new RuntimeException("Error while creating FileSystem when initializing the state of the TODBucketingSink.", e);
}
if (this.refTruncate == null) {
this.refTruncate = reflectTruncate(fs);
}
OperatorStateStore stateStore = context.getOperatorStateStore();
restoredBucketStates = stateStore.getSerializableListState("bucket-states");
int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
if (context.isRestored()) {
LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
for (State<T> recoveredState : restoredBucketStates.get()) {
handleRestoredBucketState(recoveredState);
if (LOG.isDebugEnabled()) {
LOG.debug("{} idx {} restored {}", getClass().getSimpleName(), subtaskIndex, recoveredState);
}
}
} else {
LOG.info("No state to restore for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
}
}
示例8: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkArgument(this.restoredBucketStates == null,
"The " + getClass().getSimpleName() + " has already been initialized.");
try {
initFileSystem();
} catch (IOException e) {
LOG.error("Error while creating FileSystem when initializing the state of the RollingSink.", e);
throw new RuntimeException("Error while creating FileSystem when initializing the state of the RollingSink.", e);
}
if (this.refTruncate == null) {
this.refTruncate = reflectTruncate(fs);
}
OperatorStateStore stateStore = context.getOperatorStateStore();
restoredBucketStates = stateStore.getSerializableListState("rolling-states");
int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
if (context.isRestored()) {
LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
for (BucketState bucketState : restoredBucketStates.get()) {
handleRestoredBucketState(bucketState);
}
if (LOG.isDebugEnabled()) {
LOG.debug("{} (taskIdx= {}) restored {}", getClass().getSimpleName(), subtaskIndex, bucketState);
}
} else {
LOG.info("No state to restore for the {} (taskIdx= {}).", getClass().getSimpleName(), subtaskIndex);
}
}
示例9: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkArgument(this.restoredBucketStates == null, "The operator has already been initialized.");
try {
initFileSystem();
} catch (IOException e) {
LOG.error("Error while creating FileSystem when initializing the state of the BucketingSink.", e);
throw new RuntimeException("Error while creating FileSystem when initializing the state of the BucketingSink.", e);
}
if (this.refTruncate == null) {
this.refTruncate = reflectTruncate(fs);
}
OperatorStateStore stateStore = context.getOperatorStateStore();
restoredBucketStates = stateStore.getSerializableListState("bucket-states");
int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
if (context.isRestored()) {
LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
for (State<T> recoveredState : restoredBucketStates.get()) {
handleRestoredBucketState(recoveredState);
if (LOG.isDebugEnabled()) {
LOG.debug("{} idx {} restored {}", getClass().getSimpleName(), subtaskIndex, recoveredState);
}
}
} else {
LOG.info("No state to restore for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
}
}
示例10: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkState(this.checkpointedState == null,
"The " + getClass().getSimpleName() + " has already been initialized.");
this.checkpointedState = context.getOperatorStateStore().getListState(
new ListStateDescriptor<>(
"stateful-sequence-source-state",
LongSerializer.INSTANCE
)
);
this.valuesToEmit = new ArrayDeque<>();
if (context.isRestored()) {
// upon restoring
for (Long v : this.checkpointedState.get()) {
this.valuesToEmit.add(v);
}
} else {
// the first time the job is executed
final int stepSize = getRuntimeContext().getNumberOfParallelSubtasks();
final int taskIdx = getRuntimeContext().getIndexOfThisSubtask();
final long congruence = start + taskIdx;
long totalNoOfElements = Math.abs(end - start + 1);
final int baseSize = safeDivide(totalNoOfElements, stepSize);
final int toCollect = (totalNoOfElements % stepSize > taskIdx) ? baseSize + 1 : baseSize;
for (long collected = 0; collected < toCollect; collected++) {
this.valuesToEmit.add(collected * stepSize + congruence);
}
}
}
示例11: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkState(this.checkpointedState == null,
"The " + getClass().getSimpleName() + " has already been initialized.");
this.checkpointedState = context
.getOperatorStateStore()
.getSerializableListState("message-acknowledging-source-state");
this.idsForCurrentCheckpoint = new HashSet<>(64);
this.pendingCheckpoints = new ArrayDeque<>();
this.idsProcessedButNotAcknowledged = new HashSet<>();
if (context.isRestored()) {
LOG.info("Restoring state for the {}.", getClass().getSimpleName());
List<SerializedCheckpointData[]> retrievedStates = new ArrayList<>();
for (SerializedCheckpointData[] entry : this.checkpointedState.get()) {
retrievedStates.add(entry);
}
// given that the parallelism of the function is 1, we can only have at most 1 state
Preconditions.checkArgument(retrievedStates.size() == 1,
getClass().getSimpleName() + " retrieved invalid state.");
pendingCheckpoints = SerializedCheckpointData.toDeque(retrievedStates.get(0), idSerializer);
// build a set which contains all processed ids. It may be used to check if we have
// already processed an incoming message.
for (Tuple2<Long, Set<UId>> checkpoint : pendingCheckpoints) {
idsProcessedButNotAcknowledged.addAll(checkpoint.f1);
}
} else {
LOG.info("No state to restore for the {}.", getClass().getSimpleName());
}
}
示例12: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
serializableListState = context
.getOperatorStateStore()
.getListState(new ListStateDescriptor<>("test-state", IntSerializer.INSTANCE));
if (context.isRestored()) {
Iterator<Integer> integers = serializableListState.get().iterator();
int act = integers.next();
Assert.assertEquals(42, act);
Assert.assertFalse(integers.hasNext());
wasRestored = true;
}
}
示例13: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkArgument(this.restoredBucketStates == null,
"The " + getClass().getSimpleName() + " has already been initialized.");
initFileSystem();
if (this.refTruncate == null) {
this.refTruncate = reflectTruncate(fs);
}
OperatorStateStore stateStore = context.getOperatorStateStore();
restoredBucketStates = stateStore.getSerializableListState("rolling-states");
int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
if (context.isRestored()) {
LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
for (BucketState bucketState : restoredBucketStates.get()) {
handleRestoredBucketState(bucketState);
}
if (LOG.isDebugEnabled()) {
LOG.debug("{} (taskIdx= {}) restored {}", getClass().getSimpleName(), subtaskIndex, bucketState);
}
} else {
LOG.info("No state to restore for the {} (taskIdx= {}).", getClass().getSimpleName(), subtaskIndex);
}
}
示例14: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
Preconditions.checkArgument(this.restoredBucketStates == null, "The operator has already been initialized.");
initFileSystem();
if (this.refTruncate == null) {
this.refTruncate = reflectTruncate(fs);
}
OperatorStateStore stateStore = context.getOperatorStateStore();
restoredBucketStates = stateStore.getSerializableListState("bucket-states");
int subtaskIndex = getRuntimeContext().getIndexOfThisSubtask();
if (context.isRestored()) {
LOG.info("Restoring state for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
for (State<T> recoveredState : restoredBucketStates.get()) {
handleRestoredBucketState(recoveredState);
if (LOG.isDebugEnabled()) {
LOG.debug("{} idx {} restored {}", getClass().getSimpleName(), subtaskIndex, recoveredState);
}
}
} else {
LOG.info("No state to restore for the {} (taskIdx={}).", getClass().getSimpleName(), subtaskIndex);
}
}
示例15: initializeState
import org.apache.flink.runtime.state.FunctionInitializationContext; //导入方法依赖的package包/类
/**
* The actual paths in this depend on the binary checkpoint so it you update this the paths
* here have to be updated as well.
*/
@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
OperatorStateStore stateStore = context.getOperatorStateStore();
ListState<State<T>> restoredBucketStates = stateStore.getSerializableListState("bucket-states");
if (context.isRestored()) {
for (State<T> states : restoredBucketStates.get()) {
for (String bucketPath : states.bucketStates.keySet()) {
BucketState state = states.getBucketState(new Path(bucketPath));
String current = state.currentFile;
long validLength = state.currentFileValidLength;
Assert.assertEquals(expectedBucketFilesPrefix + "4", current);
Assert.assertEquals(6, validLength);
List<String> pendingFiles = state.pendingFiles;
assertTrue(pendingFiles.isEmpty());
final Map<Long, List<String>> pendingFilesPerCheckpoint = state.pendingFilesPerCheckpoint;
Assert.assertEquals(1, pendingFilesPerCheckpoint.size());
for (Map.Entry<Long, List<String>> entry: pendingFilesPerCheckpoint.entrySet()) {
long checkpoint = entry.getKey();
List<String> files = entry.getValue();
Assert.assertEquals(0L, checkpoint);
Assert.assertEquals(4, files.size());
for (int i = 0; i < 4; i++) {
Assert.assertEquals(
expectedBucketFilesPrefix + i,
files.get(i));
}
}
}
}
}
initializeCalled = true;
super.initializeState(context);
}