Java KafkaStreams.store方法代碼示例

import org.apache.kafka.streams.KafkaStreams; //導入方法依賴的package包/類
@Override
public ReadOnlyKeyValueStore<Long, byte[]> worker() {
    Properties config = super.configBuilder()//
            .put(StreamsConfig.APPLICATION_ID_CONFIG, MallConstants.ORDER_COMMITED_TOPIC)//
            .put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap)//
            .put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.Long().getClass())//
            .put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.ByteArray().getClass())//
            .build();

    StreamsBuilder builder = new StreamsBuilder();
    KafkaStreams streams = new KafkaStreams(builder.build(), new StreamsConfig(config));
    streams.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
        // TODO Auto-generated method stub
        log.error(e.getMessage());
    });
    streams.start();

    return this.worker = // k-v query
            streams.store(queryableStoreName, QueryableStoreTypes.<Long, byte[]>keyValueStore());
}
 

import org.apache.kafka.streams.KafkaStreams; //導入方法依賴的package包/類
/**
 * Query local state store to extract metrics
 *
 * @return local Metrics
 */
private Metrics getLocalMetrics() {
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());
    KeyValueIterator<String, Double> storeIterator = averageStore.all();

    while (storeIterator.hasNext()) {
        KeyValue<String, Double> kv = storeIterator.next();
        localMetrics.add(source, kv.key, String.valueOf(kv.value));

    }
    LOGGER.log(Level.INFO, "Local store state {0}", localMetrics);
    return localMetrics;
}
 

import org.apache.kafka.streams.KafkaStreams; //導入方法依賴的package包/類
private void verifyStateStore(final KafkaStreams streams, final Set<KeyValue<Long, Long>> expectedStoreContent) {
    ReadOnlyKeyValueStore<Long, Long> store = null;

    final long maxWaitingTime = System.currentTimeMillis() + 300000L;
    while (System.currentTimeMillis() < maxWaitingTime) {
        try {
            store = streams.store(storeName, QueryableStoreTypes.<Long, Long>keyValueStore());
            break;
        } catch (final InvalidStateStoreException okJustRetry) {
            try {
                Thread.sleep(5000L);
            } catch (final Exception ignore) { }
        }
    }

    final KeyValueIterator<Long, Long> it = store.all();
    while (it.hasNext()) {
        assertTrue(expectedStoreContent.remove(it.next()));
    }

    assertTrue(expectedStoreContent.isEmpty());
}
 

import org.apache.kafka.streams.KafkaStreams; //導入方法依賴的package包/類
@Test
public void shouldBeAbleToQueryMapValuesState() throws Exception {
    streamsConfiguration.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    streamsConfiguration.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    final KStreamBuilder builder = new KStreamBuilder();
    final String[] keys = {"hello", "goodbye", "welcome", "go", "kafka"};
    final Set<KeyValue<String, String>> batch1 = new HashSet<>();
    batch1.addAll(Arrays.asList(
        new KeyValue<>(keys[0], "1"),
        new KeyValue<>(keys[1], "1"),
        new KeyValue<>(keys[2], "3"),
        new KeyValue<>(keys[3], "5"),
        new KeyValue<>(keys[4], "2")));

    IntegrationTestUtils.produceKeyValuesSynchronously(
        streamOne,
        batch1,
        TestUtils.producerConfig(
            CLUSTER.bootstrapServers(),
            StringSerializer.class,
            StringSerializer.class,
            new Properties()),
        mockTime);

    final KTable<String, String> t1 = builder.table(streamOne);
    final KTable<String, Long> t2 = t1.mapValues(new ValueMapper<String, Long>() {
        @Override
        public Long apply(final String value) {
            return Long.valueOf(value);
        }
    }, Serdes.Long(), "queryMapValues");
    t2.to(Serdes.String(), Serdes.Long(), outputTopic);

    kafkaStreams = new KafkaStreams(builder, streamsConfiguration);
    kafkaStreams.start();

    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);

    final ReadOnlyKeyValueStore<String, Long>
        myMapStore = kafkaStreams.store("queryMapValues",
        QueryableStoreTypes.<String, Long>keyValueStore());
    for (final KeyValue<String, String> batchEntry : batch1) {
        assertEquals(myMapStore.get(batchEntry.key), Long.valueOf(batchEntry.value));
    }
}
 

import org.apache.kafka.streams.KafkaStreams; //導入方法依賴的package包/類
private void verifyCanQueryState(final int cacheSizeBytes) throws java.util.concurrent.ExecutionException, InterruptedException {
    streamsConfiguration.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, cacheSizeBytes);
    final KStreamBuilder builder = new KStreamBuilder();
    final String[] keys = {"hello", "goodbye", "welcome", "go", "kafka"};

    final Set<KeyValue<String, String>> batch1 = new TreeSet<>(stringComparator);
    batch1.addAll(Arrays.asList(
        new KeyValue<>(keys[0], "hello"),
        new KeyValue<>(keys[1], "goodbye"),
        new KeyValue<>(keys[2], "welcome"),
        new KeyValue<>(keys[3], "go"),
        new KeyValue<>(keys[4], "kafka")));


    final Set<KeyValue<String, Long>> expectedCount = new TreeSet<>(stringLongComparator);
    for (final String key : keys) {
        expectedCount.add(new KeyValue<>(key, 1L));
    }

    IntegrationTestUtils.produceKeyValuesSynchronously(
            streamOne,
            batch1,
            TestUtils.producerConfig(
            CLUSTER.bootstrapServers(),
            StringSerializer.class,
            StringSerializer.class,
            new Properties()),
            mockTime);

    final KStream<String, String> s1 = builder.stream(streamOne);

    // Non Windowed
    s1.groupByKey().count("my-count").to(Serdes.String(), Serdes.Long(), outputTopic);

    s1.groupByKey().count(TimeWindows.of(WINDOW_SIZE), "windowed-count");
    kafkaStreams = new KafkaStreams(builder, streamsConfiguration);
    kafkaStreams.start();

    waitUntilAtLeastNumRecordProcessed(outputTopic, 1);

    final ReadOnlyKeyValueStore<String, Long>
        myCount = kafkaStreams.store("my-count", QueryableStoreTypes.<String, Long>keyValueStore());

    final ReadOnlyWindowStore<String, Long> windowStore =
        kafkaStreams.store("windowed-count", QueryableStoreTypes.<String, Long>windowStore());
    verifyCanGetByKey(keys,
        expectedCount,
        expectedCount,
        windowStore,
        myCount);

    verifyRangeAndAll(expectedCount, myCount);
}
 

import org.apache.kafka.streams.KafkaStreams; //導入方法依賴的package包/類
/**
 * Waits until the named store is queryable and, once it is, returns a reference to the store.
 *
 * Caveat: This is a point in time view and it may change due to partition reassignment.
 * That is, the returned store may still not be queryable in case a rebalancing is happening or
 * happened around the same time.  This caveat is acceptable for testing purposes when only a
 * single `KafkaStreams` instance of the application is running.
 *
 * @param streams            the `KafkaStreams` instance to which the store belongs
 * @param storeName          the name of the store
 * @param queryableStoreType the type of the (queryable) store
 * @param <T>                the type of the (queryable) store
 * @return the same store, which is now ready for querying (but see caveat above)
 */
public static <T> T waitUntilStoreIsQueryable(final String storeName,
                                              final QueryableStoreType<T> queryableStoreType,
                                              final KafkaStreams streams) throws InterruptedException {
  while (true) {
    try {
      return streams.store(storeName, queryableStoreType);
    } catch (InvalidStateStoreException ignored) {
      // store not yet ready for querying
      Thread.sleep(50);
    }
  }
}