Java Materialized类代码示例

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
public static void main(String[] args) throws CertificateException, NoSuchAlgorithmException,
    KeyStoreException, IOException, URISyntaxException {
  Properties streamsConfig = new AggregatorConfig().getProperties();

  final StreamsBuilder builder = new StreamsBuilder();

  final KStream<Windowed<String>, String> words =
      builder.stream(String.format("%swords", HEROKU_KAFKA_PREFIX));

  words
      .groupBy((key, word) -> word)
      .windowedBy(TimeWindows.of(TimeUnit.SECONDS.toMillis(10)))
      .count(Materialized.as("windowed-counts"))
      .toStream()
      .process(PostgresSink::new);

  final KafkaStreams streams = new KafkaStreams(builder.build(), streamsConfig);

  streams.cleanUp();
  streams.start();

  Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public SchemaKTable aggregate(final Initializer initializer,
                              final UdafAggregator aggregator,
                              final WindowExpression windowExpression,
                              final Serde<GenericRow> topicValueSerDe,
                              final String storeName) {
  final KTable aggKtable;
  if (windowExpression != null) {
    final Materialized<String, GenericRow, ?> materialized
        = Materialized.<String, GenericRow, WindowStore<Bytes, byte[]>>as(storeName)
        .withKeySerde(Serdes.String())
        .withValueSerde(topicValueSerDe);

    final KsqlWindowExpression ksqlWindowExpression = windowExpression.getKsqlWindowExpression();
    aggKtable = ksqlWindowExpression.applyAggregate(kgroupedStream, initializer, aggregator, materialized);
  } else {
    aggKtable = kgroupedStream.aggregate(initializer, aggregator, Materialized.with(Serdes.String(), topicValueSerDe));
  }
  return new SchemaKTable(schema, aggKtable, keyField, sourceSchemaKStreams, windowExpression != null,
                          SchemaKStream.Type.AGGREGATE, functionRegistry, schemaRegistryClient);

}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
@Test
public void shouldCreateTumblingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
@Test
public void shouldCreateHoppingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
    props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass());

    final StreamsBuilder builder = new StreamsBuilder();

    builder.<String, String>stream("streams-plaintext-input")
           .flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+")))
           .groupBy((key, value) -> value)
           .count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as("counts-store"))
           .toStream()
           .to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()));

    final Topology topology = builder.build();
    final KafkaStreams streams = new KafkaStreams(topology, props);
    final CountDownLatch latch = new CountDownLatch(1);

    // attach shutdown handler to catch control-c
    Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
        @Override
        public void run() {
            streams.close();
            latch.countDown();
        }
    });

    try {
        streams.start();
        latch.await();
    } catch (Throwable e) {
        System.exit(1);
    }
    System.exit(0);
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
public static void main(String[] args) throws CertificateException, NoSuchAlgorithmException,
    KeyStoreException, IOException, URISyntaxException {
  Properties streamsConfig = new AnomalyDetectorConfig().getProperties();

  final StreamsBuilder builder = new StreamsBuilder();

  final KStream<String, String> loglines =
      builder.stream( String.format("%sloglines", HEROKU_KAFKA_PREFIX));

  KStream<Windowed<String>, Long> anomalies = loglines
      .filter((key, value) -> value.contains("login failed"))
      .selectKey((key, value) -> value.split("\\|")[0])
      .groupByKey()
      .windowedBy(TimeWindows.of(TimeUnit.SECONDS.toMillis(10)))
      .count(Materialized.as("windowed-counts"))
      .toStream();

  @SuppressWarnings("unchecked")
  KStream<Windowed<String>, Long>[] branches = anomalies
      .branch(
          (key, value) -> value > 1,
          (key, value) -> value > 0
      );

  branches[0].process(AlertSink::new);
  branches[1].process(EmailSink::new);

  final KafkaStreams streams = new KafkaStreams(builder.build(), streamsConfig);

  streams.cleanUp();
  streams.start();

  Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public KTable applyAggregate(final KGroupedStream groupedStream,
                             final Initializer initializer,
                             final UdafAggregator aggregator,
                             final Materialized<String, GenericRow, ?> materialized) {
  return groupedStream.windowedBy(SessionWindows.with(sizeUnit.toMillis(gap)))
      .aggregate(initializer, aggregator, aggregator.getMerger(),
          materialized);
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public KTable applyAggregate(final KGroupedStream groupedStream,
                             final Initializer initializer,
                             final UdafAggregator aggregator,
                             final Materialized<String, GenericRow, ?> materialized) {
  return groupedStream.windowedBy(TimeWindows.of(sizeUnit.toMillis(size)))
      .aggregate(initializer, aggregator, materialized);

}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public KTable applyAggregate(KGroupedStream groupedStream, Initializer initializer, UdafAggregator aggregator, Materialized<String, GenericRow, ?> materialized) {
  return groupedStream.windowedBy(TimeWindows.of(
      sizeUnit.toMillis(size))
      .advanceBy(
          advanceByUnit.toMillis(advanceBy)))
      .aggregate(initializer, aggregator, materialized);
}
 

import org.apache.kafka.streams.kstream.Materialized; //导入依赖的package包/类
public abstract KTable applyAggregate(final KGroupedStream groupedStream,
final Initializer initializer,
final UdafAggregator aggregator,
final Materialized<String, GenericRow, ?> materialized);