Java Serde类代码示例

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
private KafkaProducer<StatEventKey, StatAggregate> buildProducer() {

        //Configure the producers
        Map<String, Object> producerProps = getProducerProps();

        Serde<StatEventKey> statKeySerde = StatEventKeySerde.instance();
        Serde<StatAggregate> statAggregateSerde = StatAggregateSerde.instance();

        try {
            return new KafkaProducer<>(
                    producerProps,
                    statKeySerde.serializer(),
                    statAggregateSerde.serializer());
        } catch (Exception e) {
            try {
                String props = producerProps.entrySet().stream()
                        .map(entry -> "  " + entry.getKey() + "=" + entry.getValue())
                        .collect(Collectors.joining("\n"));
                LOGGER.error("Error initialising kafka producer with props:\n{}",props, e);
            } catch (Exception e1) {
            }
            LOGGER.error("Error initialising kafka producer, unable to dump property values ", e);
            throw e;
        }
    }
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
/**
 * Create a context for serialization using the specified serializers and deserializers which
 * <em>must</em> match the key and value types used as parameters for this object; the state changelog topic
 * is provided to bind this serde factory to, so that future calls for serialize / deserialize do not
 * need to provide the topic name any more.
 *
 * @param topic         the topic name
 * @param keySerde      the serde for keys; cannot be null
 * @param valueSerde    the serde for values; cannot be null
 * @throws IllegalArgumentException if key or value serde is null
 */
@SuppressWarnings("unchecked")
public StateSerdes(final String topic,
                   final Serde<K> keySerde,
                   final Serde<V> valueSerde) {
    if (topic == null) {
        throw new IllegalArgumentException("topic cannot be null");
    }
    if (keySerde == null) {
        throw new IllegalArgumentException("key serde cannot be null");
    }
    if (valueSerde == null) {
        throw new IllegalArgumentException("value serde cannot be null");
    }

    this.topic = topic;
    this.keySerde = keySerde;
    this.valueSerde = valueSerde;
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
@Override
public <K, V> Map<K, V> getRangeKeyValues(String store, Class<K> keyClass, Class<V> valueClass, Serde<K> keySerde, Serde<V> valueSerde, K from, K to) {


    return execute(() -> getUriBuilder()
            .setPath(String.format("/api/v1/kv/%s", store))
            .addParameter("keySerde", keySerde.getClass().getName())
            .addParameter("valueSerde", valueSerde.getClass().getName())
            .addParameter("from", Base64.getEncoder().encodeToString(keySerde.serializer().serialize("", from)))
            .addParameter("to", Base64.getEncoder().encodeToString(keySerde.serializer().serialize("", to)))
            .build(), bytes -> {
        MultiValuedKeyValueQueryResponse resp = mapper.readValue(bytes, MultiValuedKeyValueQueryResponse.class);


        return resp.getResults().entrySet().stream()
                .map(entry -> {
                    return new Pair<K, V>(deserialize(keyClass, keySerde, entry.getKey()),
                            deserialize(valueClass, valueSerde, entry.getValue()));
                }).collect(Collectors.toMap(Pair::getKey, pair -> pair.getValue()));
    }, () -> Collections.emptyMap());

}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
private <K, V> KTable<K, V> doTable(final AutoOffsetReset offsetReset,
                                    final Serde<K> keySerde,
                                    final Serde<V> valSerde,
                                    final TimestampExtractor timestampExtractor,
                                    final String topic,
                                    final StateStoreSupplier<KeyValueStore> storeSupplier,
                                    final boolean isQueryable) {
    final String source = newName(KStreamImpl.SOURCE_NAME);
    final String name = newName(KTableImpl.SOURCE_NAME);
    final ProcessorSupplier<K, V> processorSupplier = new KTableSource<>(storeSupplier.name());

    addSource(offsetReset, source, timestampExtractor, keySerde == null ? null : keySerde.deserializer(),
            valSerde == null ? null : valSerde.deserializer(),
            topic);
    addProcessor(name, processorSupplier, source);

    final KTableImpl<K, ?, V> kTable = new KTableImpl<>(this, name, processorSupplier,
            keySerde, valSerde, Collections.singleton(source), storeSupplier.name(), isQueryable);

    addStateStore(storeSupplier, name);
    connectSourceStoreAndTopic(storeSupplier.name(), topic);

    return kTable;
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void init(ProcessorContext context, StateStore root) {
    inner.init(context, root);

    // construct the serde
    StateSerdes<K, V>  serdes = new StateSerdes<>(
        ProcessorStateManager.storeChangelogTopic(context.applicationId(), inner.name()),
        keySerde == null ? (Serde<K>) context.keySerde() : keySerde,
        valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde);

    this.changeLogger = new StoreChangeLogger<>(inner.name(), context, serdes);

    // if the inner store is an LRU cache, add the eviction listener to log removed record
    if (inner instanceof MemoryLRUCache) {
        ((MemoryLRUCache<K, V>) inner).whenEldestRemoved(new MemoryNavigableLRUCache.EldestEntryRemovalListener<K, V>() {
            @Override
            public void apply(K key, V value) {
                removed(key);
            }
        });
    }
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void to(final Serde<K> keySerde, final Serde<V> valSerde, StreamPartitioner<? super K, ? super V> partitioner, final String topic) {
    Objects.requireNonNull(topic, "topic can't be null");
    final String name = topology.newName(SINK_NAME);

    final Serializer<K> keySerializer = keySerde == null ? null : keySerde.serializer();
    final Serializer<V> valSerializer = valSerde == null ? null : valSerde.serializer();

    if (partitioner == null && keySerializer != null && keySerializer instanceof WindowedSerializer) {
        final WindowedSerializer<Object> windowedSerializer = (WindowedSerializer<Object>) keySerializer;
        partitioner = (StreamPartitioner<K, V>) new WindowedStreamPartitioner<Object, V>(topic, windowedSerializer);
    }

    topology.addSink(name, topic, keySerializer, valSerializer, partitioner, this.name);
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
public KStreamTestDriver(final KStreamBuilder builder,
                         final File stateDir,
                         final Serde<?> keySerde,
                         final Serde<?> valSerde,
                         final long cacheSize) {
    builder.setApplicationId("TestDriver");
    topology = builder.build(null);
    globalTopology = builder.buildGlobalStateTopology();
    final ThreadCache cache = new ThreadCache("testCache", cacheSize, new MockStreamsMetrics(new Metrics()));
    context = new MockProcessorContext(stateDir, keySerde, valSerde, new MockRecordCollector(), cache);
    context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "topic"));
    // init global topology first as it will add stores to the
    // store map that are required for joins etc.
    if (globalTopology != null) {
        initTopology(globalTopology, globalTopology.globalStateStores());
    }
    initTopology(topology, topology.stateStores());
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
private <V1> KTable<K, V1> doMapValues(final ValueMapper<? super V, ? extends V1> mapper,
                                       final Serde<V1> valueSerde,
                                       final StateStoreSupplier<KeyValueStore> storeSupplier) {
    Objects.requireNonNull(mapper);
    String name = topology.newName(MAPVALUES_NAME);
    String internalStoreName = null;
    if (storeSupplier != null) {
        internalStoreName = storeSupplier.name();
    }
    KTableProcessorSupplier<K, V, V1> processorSupplier = new KTableMapValues<>(this, mapper, internalStoreName);
    topology.addProcessor(name, processorSupplier, this.name);
    if (storeSupplier != null) {
        topology.addStateStore(storeSupplier, name);
        return new KTableImpl<>(topology, name, processorSupplier, this.keySerde, valueSerde, sourceNodes, internalStoreName, true);
    } else {
        return new KTableImpl<>(topology, name, processorSupplier, sourceNodes, this.queryableStoreName, false);
    }
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
/**
 * Return an {@link Serde#configure(Map, boolean) configured} instance of {@link #VALUE_SERDE_CLASS_CONFIG value
 * Serde class}. This method is deprecated. Use {@link #defaultValueSerde()} instead.
 *
 * @return an configured instance of value Serde class
 */
@Deprecated
public Serde valueSerde() {
    try {
        Serde<?> serde = getConfiguredInstance(VALUE_SERDE_CLASS_CONFIG, Serde.class);
        // the default value of deprecated value serde is null
        if (serde == null) {
            serde = defaultValueSerde();
        } else {
            serde.configure(originals(), false);
        }
        return serde;
    } catch (final Exception e) {
        throw new StreamsException(String.format("Failed to configure value serde %s", get(VALUE_SERDE_CLASS_CONFIG)), e);
    }
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public <T> KTable<Windowed<K>, T> aggregate(final Initializer<T> initializer,
                                            final Aggregator<? super K, ? super V, T> aggregator,
                                            final Merger<? super K, T> sessionMerger,
                                            final SessionWindows sessionWindows,
                                            final Serde<T> aggValueSerde,
                                            final String queryableStoreName) {
    determineIsQueryable(queryableStoreName);
    return aggregate(initializer,
                     aggregator,
                     sessionMerger,
                     sessionWindows,
                     aggValueSerde,
                     storeFactory(keySerde, aggValueSerde, getOrCreateName(queryableStoreName, AGGREGATE_NAME))
                      .sessionWindowed(sessionWindows.maintainMs()).build());


}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
/**
 * Creates a typical word count topology
 */
private KafkaStreams createCountStream(final String inputTopic, final String outputTopic, final Properties streamsConfiguration) {
    final KStreamBuilder builder = new KStreamBuilder();
    final Serde<String> stringSerde = Serdes.String();
    final KStream<String, String> textLines = builder.stream(stringSerde, stringSerde, inputTopic);

    final KGroupedStream<String, String> groupedByWord = textLines
        .flatMapValues(new ValueMapper<String, Iterable<String>>() {
            @Override
            public Iterable<String> apply(final String value) {
                return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"));
            }
        })
        .groupBy(MockKeyValueMapper.<String, String>SelectValueMapper());

    // Create a State Store for the all time word count
    groupedByWord.count("word-count-store-" + inputTopic).to(Serdes.String(), Serdes.Long(), outputTopic);

    // Create a Windowed State Store that contains the word count for every 1 minute
    groupedByWord.count(TimeWindows.of(WINDOW_SIZE), "windowed-word-count-store-" + inputTopic);

    return new KafkaStreams(builder, streamsConfiguration);
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private void initInternal(final InternalProcessorContext context) {
    this.context = context;

    keySchema.init(topic);
    serdes = new StateSerdes<>(
        topic,
        keySerde == null ? (Serde<K>) context.keySerde() : keySerde,
        aggSerde == null ? (Serde<AGG>) context.valueSerde() : aggSerde);


    cacheName = context.taskId() + "-" + bytesStore.name();
    cache = context.getCache();
    cache.addDirtyEntryFlushListener(cacheName, new ThreadCache.DirtyEntryFlushListener() {
        @Override
        public void apply(final List<ThreadCache.DirtyEntry> entries) {
            for (ThreadCache.DirtyEntry entry : entries) {
                putAndMaybeForward(entry, context);
            }
        }
    });
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
/**
 * Helper method for testing that a serde can serialize its object and deserialize it
 * back again. Throws a {@link RuntimeException } if the objects don't match.
 * @param serde
 * @param obj
 * @param <T>
 */
public static <T> void verify(final Serde<T> serde, final T obj) {
    final String dummyTopic = "xxx";
    byte[] bytes = serde.serializer().serialize(dummyTopic, obj);

    LOGGER.trace(() -> String.format("object form: %s", obj));
    LOGGER.trace(() -> String.format("byte form: %s", ByteArrayUtils.byteArrayToHex(bytes)));

    T deserializedObj = serde.deserializer().deserialize(dummyTopic, bytes);
    if (!obj.equals(deserializedObj)) {
        throw new RuntimeException(String.format("Original [%s] and de-serialized [%s] values don't match", obj, deserializedObj));
    }
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
protected Serde<Object> getSerde(String serde){
    try {
        return (Serde<Object>) Class.forName(serde).newInstance();

    } catch (InstantiationException | IllegalAccessException | ClassNotFoundException | ClassCastException e) {
        throw new SerdeNotFoundException(serde, e);
    }
}
 

import org.apache.kafka.common.serialization.Serde; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void init(final ProcessorContext context, final StateStore root) {
    this.context = context;
    // construct the serde
    serdes = new StateSerdes<>(ProcessorStateManager.storeChangelogTopic(context.applicationId(), bytesStore.name()),
                               keySerde == null ? (Serde<K>) context.keySerde() : keySerde,
                               valueSerde == null ? (Serde<V>) context.valueSerde() : valueSerde);

    bytesStore.init(context, root);
}