From abb7b26a1fb8267e509bc304687052aeb7b9acb6 Mon Sep 17 00:00:00 2001 From: Kai Moritz Date: Sat, 14 Mar 2026 17:37:00 +0100 Subject: [PATCH] `simple-producer` in `simple-consumer` verwandelt -- ALIGN --- Dockerfile | 2 +- README.sh | 15 +- docker/docker-compose.yml | 4 + pom.xml | 8 +- settings.gradle | 2 +- .../java/de/juplo/kafka/ExampleConsumer.java | 170 ++++++++---------- 6 files changed, 97 insertions(+), 104 deletions(-) diff --git a/Dockerfile b/Dockerfile index 74e66edf..22819afe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,4 +3,4 @@ VOLUME /tmp COPY target/*.jar /opt/app.jar COPY target/libs /opt/libs ENTRYPOINT [ "java", "-jar", "/opt/app.jar" ] -CMD [ "kafka:9092", "test", "DCKR" ] +CMD [ "kafka:9092", "test", "my-group", "DCKR" ] diff --git a/README.sh b/README.sh index 3d98ace7..85b8f960 100755 --- a/README.sh +++ b/README.sh @@ -1,6 +1,6 @@ #!/bin/bash -IMAGE=juplo/simple-producer:1.0-SNAPSHOT +IMAGE=juplo/simple-consumer:1.0-SNAPSHOT if [ "$1" = "cleanup" ] then @@ -10,7 +10,7 @@ then fi docker compose -f docker/docker-compose.yml up -d --remove-orphans kafka-1 kafka-2 kafka-3 -docker compose -f docker/docker-compose.yml rm -svf producer +docker compose -f docker/docker-compose.yml rm -svf consumer if [[ $(docker image ls -q $IMAGE) == "" || @@ -27,10 +27,13 @@ docker compose -f docker/docker-compose.yml up --remove-orphans setup || exit 1 docker compose -f docker/docker-compose.yml up -d producer +docker compose -f docker/docker-compose.yml up -d consumer + sleep 5 +docker compose -f docker/docker-compose.yml stop consumer -docker compose -f docker/docker-compose.yml exec cli kafkacat -b kafka:9092 -t test -c 20 -f'topic=%t\tpartition=%p\toffset=%o\tkey=%k\tvalue=%s\n' +docker compose -f docker/docker-compose.yml start consumer +sleep 5 -docker compose -f docker/docker-compose.yml stop producer -docker compose -f docker/docker-compose.yml exec cli kafkacat -b kafka:9092 -t test -e -f'topic=%t\tpartition=%p\toffset=%o\tkey=%k\tvalue=%s\n' -docker compose -f docker/docker-compose.yml logs producer +docker compose -f docker/docker-compose.yml stop producer consumer +docker compose -f docker/docker-compose.yml logs consumer diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2ab5b6cd..b1fbf6f9 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -179,6 +179,10 @@ services: cpu_quota: 50000 mem_limit: 100m + consumer: + image: juplo/simple-consumer:1.0-SNAPSHOT + command: kafka:9092 test my-group consumer + volumes: controller-data: kafka-1-data: diff --git a/pom.xml b/pom.xml index 2a04e375..9827f688 100644 --- a/pom.xml +++ b/pom.xml @@ -12,9 +12,9 @@ de.juplo.kafka - simple-producer - Super Simple Producer - A Simple Producer, programmed with pure Java, that sends messages via Kafka + simple-consumer + Simple Consumer-Group + Super Simple Consumer-Group, that is implemented as a plain Java-program 1.0-SNAPSHOT @@ -63,7 +63,7 @@ true libs/ - de.juplo.kafka.ExampleProducer + de.juplo.kafka.ExampleConsumer diff --git a/settings.gradle b/settings.gradle index 5f64b227..71fb5737 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1 +1 @@ -rootProject.name = 'simple-producer' +rootProject.name = 'simple-consumer' diff --git a/src/main/java/de/juplo/kafka/ExampleConsumer.java b/src/main/java/de/juplo/kafka/ExampleConsumer.java index a7645889..534fb052 100644 --- a/src/main/java/de/juplo/kafka/ExampleConsumer.java +++ b/src/main/java/de/juplo/kafka/ExampleConsumer.java @@ -1,143 +1,123 @@ package de.juplo.kafka; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringSerializer; - +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.apache.kafka.clients.consumer.ConsumerRecords; +import org.apache.kafka.clients.consumer.Consumer; +import org.apache.kafka.clients.consumer.KafkaConsumer; +import org.apache.kafka.common.errors.WakeupException; +import org.apache.kafka.common.serialization.StringDeserializer; + +import java.time.Duration; +import java.util.Arrays; import java.util.Properties; @Slf4j -public class ExampleProducer +public class ExampleConsumer { private final String id; private final String topic; - private final Producer producer; + private final Consumer consumer; - private volatile boolean running = true; - private volatile boolean done = false; - private long produced = 0; + private volatile boolean running = false; + private long consumed = 0; - public ExampleProducer( + public ExampleConsumer( String broker, String topic, + String groupId, String clientId) { Properties props = new Properties(); props.put("bootstrap.servers", broker); + props.put("group.id", groupId); // ID für die Offset-Commits props.put("client.id", clientId); // Nur zur Wiedererkennung - props.put("key.serializer", StringSerializer.class.getName()); - props.put("value.serializer", StringSerializer.class.getName()); + props.put("auto.offset.reset", "earliest"); // Von Beginn an lesen + props.put("partition.assignment.strategy", "org.apache.kafka.clients.consumer.CooperativeStickyAssignor"); + props.put("key.deserializer", StringDeserializer.class.getName()); + props.put("value.deserializer", StringDeserializer.class.getName()); this.id = clientId; this.topic = topic; - producer = new KafkaProducer<>(props); + consumer = new KafkaConsumer<>(props); } + public void run() { - long i = 0; - try { - for (; running; i++) + log.info("{} - Subscribing to topic {}", id, topic); + consumer.subscribe(Arrays.asList(topic)); + running = true; + + while (true) { - send(Long.toString(i%10), Long.toString(i)); - Thread.sleep(500); + ConsumerRecords records = consumer.poll(Duration.ofSeconds(1)); + + log.info("{} - Received {} messages", id, records.count()); + for (ConsumerRecord record : records) + { + consumed++; + log.info( + "{} - {}: {}/{} - {}={}", + id, + record.offset(), + record.topic(), + record.partition(), + record.key(), + record.value() + ); + } } } - catch (Exception e) + catch(WakeupException e) { - log.error("{} - Unexpected error!", id, e); + log.info("{} - Consumer was signaled to finish its work", id); } - finally + catch(Exception e) { - log.info("{}: Closing the KafkaProducer", id); - producer.close(); - log.info("{}: Produced {} messages in total, exiting!", id, produced); - done = true; + log.error("{} - Unexpected error, unsubscribing!", id, e); + consumer.unsubscribe(); } - } - - void send(String key, String value) - { - final long sendRequested = System.currentTimeMillis(); - - final ProducerRecord record = new ProducerRecord<>( - topic, // Topic - key, // Key - value // Value - ); - - producer.send(record, (metadata, e) -> + finally { - long sendRequestProcessed = System.currentTimeMillis(); - if (e == null) - { - // HANDLE SUCCESS - log.debug( - "{} - Sent message {}={}, partition={}, offset={}, timestamp={}, latency={}ms", - id, - key, - value, - metadata.partition(), - metadata.offset(), - metadata.timestamp(), - sendRequestProcessed - sendRequested - ); - } - else - { - // HANDLE ERROR - log.error( - "{} - ERROR for message {}={}, latency={}ms: {}", - id, - key, - value, - sendRequestProcessed - sendRequested, - e.toString() - ); - } - }); - - long sendRequestQueued = System.currentTimeMillis(); - produced++; - log.trace( - "{} - Queued message {}={}, latency={}ms", - id, - key, - value, - sendRequestQueued - sendRequested - ); + running = false; + log.info("{} - Closing the KafkaConsumer", id); + consumer.close(); + log.info("{}: Consumed {} messages in total, exiting!", id, consumed); + } } public static void main(String[] args) throws Exception { - if (args.length != 3) + String broker = ":9092"; + String topic = "test"; + String groupId = "my-group"; + String clientId = "DEV"; + + switch (args.length) { - log.error("Three arguments required!"); - log.error("arg[0]: Broker-Address"); - log.error("arg[1]: Topic"); - log.error("arg[2]: Unique Client-ID"); - System.exit(1); - return; + case 4: + clientId = args[3]; + case 3: + groupId = args[2]; + case 2: + topic = args[1]; + case 1: + broker = args[0]; } - log.info( - "Running ExampleProducer: broker={}, topic={}, client-id={}", - args[0], - args[1], - args[2]); - ExampleProducer instance = new ExampleProducer(args[0], args[1], args[2]); + ExampleConsumer instance = new ExampleConsumer(broker, topic, groupId, clientId); Runtime.getRuntime().addShutdownHook(new Thread(() -> { - instance.running = false; - while (!instance.done) + instance.consumer.wakeup(); + + while (instance.running) { log.info("{} - Waiting for main-thread...", instance.id); try @@ -149,6 +129,12 @@ public class ExampleProducer log.info("{} - Shutdown completed.", instance.id); })); + log.info( + "Running ExampleConsumer: broker={}, topic={}, group-id={}, client-id={}", + broker, + topic, + groupId, + clientId); instance.run(); } } -- 2.39.5