package de.juplo.kafka.wordcount.counter;
-import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KafkaStreams;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.StreamsBuilder;
+import org.apache.kafka.streams.Topology;
import org.apache.kafka.streams.kstream.KStream;
import org.apache.kafka.streams.kstream.Materialized;
import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier;
+import org.apache.kafka.streams.kstream.*;
+import org.springframework.kafka.support.serializer.JsonSerde;
import java.util.Properties;
Properties properties,
KeyValueBytesStoreSupplier storeSupplier,
ObjectMapper mapper)
+ {
+ Topology topology = CounterStreamProcessor.buildTopology(
+ inputTopic,
+ outputTopic,
+ storeSupplier,
+ mapper);
+
+ streams = new KafkaStreams(topology, properties);
+ }
+
+ static Topology buildTopology(
+ String inputTopic,
+ String outputTopic,
+ KeyValueBytesStoreSupplier storeSupplier,
+ ObjectMapper mapper)
{
StreamsBuilder builder = new StreamsBuilder();
- KStream<String, String> source = builder.stream(inputTopic);
+ KStream<String, Word> source = builder.stream(
+ inputTopic,
+ Consumed.with(
+ Serdes.String(),
+ new JsonSerde<>(Word.class)
+ .ignoreTypeHeaders()));
+
source
- .map((username, word) ->
- {
- try
- {
- String key = mapper.writeValueAsString(Key.of(username, word));
- return new KeyValue<>(key, word);
- }
- catch (JsonProcessingException e)
- {
- throw new RuntimeException(e);
- }
- })
- .groupByKey()
- .count(Materialized.as(storeSupplier))
- .mapValues(value->Long.toString(value))
+ .map((key, word) -> new KeyValue<>(word, word))
+ .groupByKey(Grouped.with(
+ new JsonSerde<>(Word.class)
+ .forKeys()
+ .noTypeInfo(),
+ new JsonSerde<>(Word.class)
+ .noTypeInfo()))
+ .count(Materialized
+ .<Word,Long>as(storeSupplier)
+ .withKeySerde(
+ new JsonSerde<>(Word.class)
+ .forKeys()
+ .noTypeInfo())
+ .withValueSerde(
+ Serdes.Long()))
.toStream()
- .to(outputTopic);
+ .map((word, count) -> new KeyValue<>(word, WordCount.of(word.getUser(), word.getWord(), count)))
+ .to(
+ outputTopic,
+ Produced.with(
+ new JsonSerde<>(Word.class)
+ .forKeys()
+ .noTypeInfo(),
+ new JsonSerde<>(WordCount.class)
+ .noTypeInfo()));
+
+ Topology topology = builder.build();
+ log.info("\n\n{}", topology.describe());
- streams = new KafkaStreams(builder.build(), properties);
+ return topology;
}
public void start()