Java ReaderGroupConfig类代码示例

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
/**
 * Create a stream reader for reading Integer events.
 *
 * @param streamName    Name of the test stream.
 *
 * @return Stream reader instance.
 */
public EventStreamReader<Integer> getIntegerReader(final String streamName) {
    Preconditions.checkState(this.started.get(), "Services not yet started");
    Preconditions.checkNotNull(streamName);

    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(this.scope, getControllerUri());
    final String readerGroup = "testReaderGroup" + this.scope + streamName;
    readerGroupManager.createReaderGroup(
            readerGroup,
            ReaderGroupConfig.builder().startingTime(0).build(),
            Collections.singleton(streamName));

    ClientFactory clientFactory = ClientFactory.withScope(this.scope, getControllerUri());
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(
            readerGroupId,
            readerGroup,
            new IntegerSerializer(),
            ReaderConfig.builder().build());
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
void initialize() throws CheckpointStoreException {

        try {
            checkpointStore.addReaderGroup(actorSystem.getProcess(), eventProcessorConfig.getConfig().getReaderGroupName());
        } catch (CheckpointStoreException e) {
            if (!e.getType().equals(CheckpointStoreException.Type.NodeExists)) {
                throw e;
            } else {
                log.warn("reader group {} exists", eventProcessorConfig.getConfig().getReaderGroupName());
            }
        }

        // Continue creating reader group if adding reader group to checkpoint store succeeds.

        readerGroup = createIfNotExists(
                actorSystem.readerGroupManager,
                eventProcessorConfig.getConfig().getReaderGroupName(),
                ReaderGroupConfig.builder().disableAutomaticCheckpoints().startingPosition(Sequence.MIN_VALUE).build(),
                Collections.singleton(eventProcessorConfig.getConfig().getStreamName()));

        createEventProcessors(eventProcessorConfig.getConfig().getEventProcessorCount() - eventProcessorMap.values().size());
    }
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private EventProcessorSystemImpl createMockSystem(final String name, final String processId, final String scope,
                                                  final SequenceAnswer<EventStreamReader<TestEvent>> readers,
                                                  final EventStreamWriter<TestEvent> writer,
                                                  final String readerGroupName) {
    ClientFactory clientFactory = Mockito.mock(ClientFactory.class);
    Mockito.when(clientFactory.createReader(anyString(), anyString(), any(), any()))
            .thenAnswer(readers);

    Mockito.when(clientFactory.<TestEvent>createEventWriter(anyString(), any(), any())).thenReturn(writer);

    ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
    Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);

    ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
    Mockito.when(readerGroupManager.createReaderGroup(anyString(), any(ReaderGroupConfig.class), any()))
            .then(invocation -> readerGroup);

    return new EventProcessorSystemImpl(name, processId, scope, clientFactory, readerGroupManager);
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    @Cleanup
    MockStreamManager streamManager = new MockStreamManager(StartLocalService.SCOPE,
                                                            InetAddress.getLocalHost().getHostAddress(),
                                                            StartLocalService.SERVICE_PORT);
    streamManager.createScope(StartLocalService.SCOPE);
    streamManager.createStream(StartLocalService.SCOPE, StartLocalService.STREAM_NAME, null);
    streamManager.createReaderGroup(READER_GROUP,
                                    ReaderGroupConfig.builder().startingTime(0).build(),
                                    Collections.singleton(StartLocalService.STREAM_NAME));
    EventStreamReader<String> reader = streamManager.getClientFactory().createReader(UUID.randomUUID().toString(),
                                                                                     READER_GROUP,
                                                                                     new JavaSerializer<>(),
                                                                                     ReaderConfig.builder().build());
    for (int i = 0; i < 20; i++) {
        String event = reader.readNextEvent(60000).getEvent();
        System.err.println("Read event: " + event);
    }
    reader.close();
    System.exit(0);
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
/**
 * Create a stream reader for reading Integer events.
 *
 * @param streamName    Name of the test stream.
 *
 * @return Stream reader instance.
 */
public EventStreamReader<Integer> getIntegerReader(final String streamName) {
    Preconditions.checkState(this.started.get(), "Services not yet started");
    Preconditions.checkNotNull(streamName);

    ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(this.scope, this.controllerUri);
    final String readerGroup = "testReaderGroup" + this.scope + streamName;
    readerGroupManager.createReaderGroup(
            readerGroup,
            ReaderGroupConfig.builder().startingTime(0).build(),
            Collections.singleton(streamName));

    ClientFactory clientFactory = ClientFactory.withScope(this.scope, this.controllerUri);
    final String readerGroupId = UUID.randomUUID().toString();
    return clientFactory.createReader(
            readerGroupId,
            readerGroup,
            new IntegerSerializer(),
            ReaderConfig.builder().build());
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
/**
 * Creates a new Flink Pravega reader instance which can be added as a source to a Flink job.
 *
 * <p>The reader will use the given {@code readerName} to store its state (its positions
 * in the stream segments) in Flink's checkpoints/savepoints. This name is used in a similar
 * way as the operator UIDs ({@link SingleOutputStreamOperator#uid(String)}) to identify state
 * when matching it into another job that resumes from this job's checkpoints/savepoints.
 *
 * <p>Without specifying a {@code readerName}, the job will correctly checkpoint and recover,
 * but new instances of the job can typically not resume this reader's state (positions).
 *
 * @param controllerURI         The pravega controller endpoint address.
 * @param scope                 The destination stream's scope name.
 * @param streamNames           The list of stream names to read events from.
 * @param startTime             The start time from when to read events from.
 *                              Use 0 to read all stream events from the beginning.
 * @param deserializationSchema The implementation to deserialize events from pravega streams.
 * @param readerName            The name of the reader, used to store state and resume existing
 *                              state from savepoints.
 */
public FlinkPravegaReader(final URI controllerURI, final String scope, final Set<String> streamNames,
                          final long startTime, final DeserializationSchema<T> deserializationSchema,
                          final String readerName) {

    Preconditions.checkNotNull(controllerURI, "controllerURI");
    Preconditions.checkNotNull(scope, "scope");
    Preconditions.checkNotNull(streamNames, "streamNames");
    Preconditions.checkArgument(startTime >= 0, "start time must be >= 0");
    Preconditions.checkNotNull(deserializationSchema, "deserializationSchema");
    if (readerName == null) {
        this.readerName = getDefaultReaderName(scope, streamNames);
    } else {
        this.readerName = readerName;
    }

    this.controllerURI = controllerURI;
    this.scopeName = scope;
    this.deserializationSchema = deserializationSchema;
    this.readerGroupName = generateRandomReaderGroupName();

    // TODO: This will require the client to have access to the pravega controller and handle any temporary errors.
    //       See https://github.com/pravega/pravega/issues/553.
    log.info("Creating reader group: {} for the Flink job", this.readerGroupName);

    ReaderGroupConfig groupConfig = ReaderGroupConfig.builder()
            .startingTime(startTime)
            .disableAutomaticCheckpoints()
            .build();

    ReaderGroupManager.withScope(scope, controllerURI)
            .createReaderGroup(this.readerGroupName, groupConfig, streamNames);
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
public void run() {
    StreamManager streamManager = StreamManager.create(controllerURI);
    
    final boolean scopeIsNew = streamManager.createScope(scope);
    StreamConfiguration streamConfig = StreamConfiguration.builder()
            .scalingPolicy(ScalingPolicy.fixed(1))
            .build();
    final boolean streamIsNew = streamManager.createStream(scope, streamName, streamConfig);

    final String readerGroup = UUID.randomUUID().toString().replace("-", "");
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder()
            .build();
    try (ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI)) {
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig, Collections.singleton(streamName));
    }

    try (ClientFactory clientFactory = ClientFactory.withScope(scope, controllerURI);
         EventStreamReader<String> reader = clientFactory.createReader("reader",
                                                                       readerGroup,
                                                                       new JavaSerializer<String>(),
                                                                       ReaderConfig.builder().build())) {
        System.out.format("Reading all the events from %s/%s%n", scope, streamName);
        EventRead<String> event = null;
        do {
            try {
                event = reader.readNextEvent(READER_TIMEOUT_MS);
                if (event.getEvent() != null) {
                    System.out.format("Read event '%s'%n", event.getEvent());
                }
            } catch (ReinitializationRequiredException e) {
                //There are certain circumstances where the reader needs to be reinitialized
                e.printStackTrace();
            }
        } while (event.getEvent() != null);
        System.out.format("No more events from %s/%s%n", scope, streamName);
    }
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
public void run() {
    setRunning(true);
    final String readerGroup = UUID.randomUUID().toString().replace("-", "");
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder().startingPosition(Sequence.MIN_VALUE)
                                                                 .build();

    try (ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI)) {
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig, Collections.singleton(streamName));
    }

    try (ClientFactory clientFactory = ClientFactory.withScope(scope, controllerURI);
         EventStreamReader<T> reader = clientFactory.createReader("reader",
                 readerGroup, serializer, ReaderConfig.builder().build())) {

        while (isRunning()) {
            try {
                EventRead<T> event = reader.readNextEvent(READER_TIMEOUT_MS);
                T eventData = event.getEvent();
                if (eventData != null) {
                    onNext.accept(event.getEvent());
                }
            }
            catch (ReinitializationRequiredException e) {
                onError.accept(e);
            }
        }
    }
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
public void run() {
    final String readerGroup = UUID.randomUUID().toString().replace("-", "");
    final ReaderGroupConfig readerGroupConfig = ReaderGroupConfig.builder().startingPosition(Sequence.MIN_VALUE)
            .build();
    StreamManager streamManager = StreamManager.create(controllerURI);
    streamManager.createScope(scope);

    StreamConfiguration streamConfig = StreamConfiguration.builder().scope(scope).streamName(streamName)
            .scalingPolicy(ScalingPolicy.fixed(1))
            .build();

    streamManager.createStream(scope, streamName, streamConfig);

    try (ReaderGroupManager readerGroupManager = ReaderGroupManager.withScope(scope, controllerURI)) {
        readerGroupManager.createReaderGroup(readerGroup, readerGroupConfig, Collections.singleton(streamName));
    }

    try (ClientFactory clientFactory = ClientFactory.withScope(scope, controllerURI);
         EventStreamReader<String> reader = clientFactory.createReader("reader",
                                                                       readerGroup,
                                                                       new JavaSerializer<String>(),
                                                                       ReaderConfig.builder().build())) {
        System.out.format("******** Reading events from %s/%s%n", scope, streamName);
        EventRead<String> event = null;
        do {
            try {
                event = reader.readNextEvent(READER_TIMEOUT_MS);
                if(event != null) {
                    System.out.format("'%s'%n", event.getEvent());
                }
            } catch (ReinitializationRequiredException e) {
                //There are certain circumstances where the reader needs to be reinitialized
                e.printStackTrace();
            }
        }while(true);
    }
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
private <T extends ControllerEvent>
void createEventProcessor(final String readerGroupName,
                          final String streamName,
                          final EventStreamReader<T> reader,
                          final EventStreamWriter<T> writer,
                          Supplier<EventProcessor<T>> factory) throws CheckpointStoreException {
    ClientFactory clientFactory = Mockito.mock(ClientFactory.class);
    Mockito.when(clientFactory.<T>createReader(anyString(), anyString(), any(), any())).thenReturn(reader);
    Mockito.when(clientFactory.<T>createEventWriter(anyString(), any(), any())).thenReturn(writer);

    ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
    Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);

    ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
    Mockito.when(readerGroupManager.createReaderGroup(anyString(), any(ReaderGroupConfig.class), any()))
            .then(invocation -> readerGroup);

    EventProcessorSystemImpl system = new EventProcessorSystemImpl("system", "host", SCOPE, clientFactory, readerGroupManager);

    EventProcessorGroupConfig eventProcessorConfig = EventProcessorGroupConfigImpl.builder()
            .eventProcessorCount(1)
            .readerGroupName(readerGroupName)
            .streamName(streamName)
            .checkpointConfig(CheckpointConfig.periodic(1, 1))
            .build();

    EventProcessorConfig<T> config = EventProcessorConfig.<T>builder()
            .config(eventProcessorConfig)
            .decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER)
            .serializer(new JavaSerializer<>())
            .supplier(factory)
            .build();

    system.createEventProcessorGroup(config, CheckpointStoreFactory.createInMemoryStore());
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
/**
 * Called by the StreamManager to provide the streams the group should start reading from.
 * @param  config The configuration for the reader group.
 * @param streams The segments to use and where to start from.
 */
@VisibleForTesting
public void initializeGroup(ReaderGroupConfig config, Set<String> streams) {
    @Cleanup
    StateSynchronizer<ReaderGroupState> synchronizer = createSynchronizer();
    Map<Segment, Long> segments = getSegmentsForStreams(streams);
    ReaderGroupStateManager.initializeReaderGroup(synchronizer, config, segments);
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
@Override
public void resetReadersToCheckpoint(Checkpoint checkpoint) {
    @Cleanup
    StateSynchronizer<ReaderGroupState> synchronizer = createSynchronizer();
    synchronizer.updateState(state -> {
        ReaderGroupConfig config = state.getConfig();
        Map<Segment, Long> positions = new HashMap<>();
        for (StreamCut cut : checkpoint.asImpl().getPositions().values()) {
            positions.putAll(cut.getPositions());
        }
        return Collections.singletonList(new ReaderGroupStateInit(config, positions));
    });
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
@Override
public void updateConfig(ReaderGroupConfig config, Set<String> streamNames) {
    @Cleanup
    StateSynchronizer<ReaderGroupState> synchronizer = createSynchronizer();
    Map<Segment, Long> segments = getSegmentsForStreams(streamNames);
    synchronizer.updateStateUnconditionally(new ReaderGroupStateInit(config, segments));
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
ReaderGroupState(String scopedSynchronizerStream, Revision revision, ReaderGroupConfig config, Map<Segment, Long> segmentsToOffsets) {
    Exceptions.checkNotNullOrEmpty(scopedSynchronizerStream, "scopedSynchronizerStream");
    Preconditions.checkNotNull(revision);
    Preconditions.checkNotNull(config);
    Exceptions.checkNotNullOrEmpty(segmentsToOffsets.entrySet(), "segmentsToOffsets");
    this.scopedSynchronizerStream = scopedSynchronizerStream;
    this.revision = revision;
    this.config = config;
    this.unassignedSegments = new LinkedHashMap<>(segmentsToOffsets);
}
 

import io.pravega.client.stream.ReaderGroupConfig; //导入依赖的package包/类
@Override
public ReaderGroup createReaderGroup(String groupName, ReaderGroupConfig config, Set<String> streams) {
    log.info("Creating reader group: {} for streams: {} with configuration: {}", groupName, Arrays.toString(streams.toArray()), config);
    NameUtils.validateReaderGroupName(groupName);
    createStreamHelper(getStreamForReaderGroup(groupName), StreamConfiguration.builder()
                                                                              .scope(scope)
                                                                              .streamName(getStreamForReaderGroup(groupName))
                                                                              .scalingPolicy(ScalingPolicy.fixed(1))
                                                                              .build());
    SynchronizerConfig synchronizerConfig = SynchronizerConfig.builder().build();
    ReaderGroupImpl result = new ReaderGroupImpl(scope, groupName, synchronizerConfig, new JavaSerializer<>(),
                                                 new JavaSerializer<>(), clientFactory, controller, connectionFactory);
    result.initializeGroup(config, streams);
    return result;
}