From: Kai Moritz Date: Sat, 4 Jun 2022 09:41:19 +0000 (+0200) Subject: Springify: Testfall repariert - Seltsames Verhalten von `@KafkaHandler`! X-Git-Url: http://juplo.de/gitweb/?a=commitdiff_plain;h=295373025f1d167e31704d8474c332c9441c9b1b;p=demos%2Fkafka%2Ftraining Springify: Testfall repariert - Seltsames Verhalten von `@KafkaHandler`! * Um den Testfall zu reparieren, musste die Implementierung so angepasst werden, dass die Handler wie zuvor die `RecordMetadata` von Kafka erhalten. * Dafür nehmen die `receive`-Methoden von `EndlessConsumer` und die Handler, die von diesen aufgerufen werden jetzt zwei Parameter entgegen: * Die deserialisierte Nachricht (`ClientMessage` oder `Greeting`) * Die `ConsumerRecordMetaData` * Wenn die Methoden den zweiten Parameter annehmen, werden die Nachrichten von Spring Kafka den Handler-Methoden nicht mehr korrekt übergeben: Es kommt zu einem Fehler wegen einem angeblich fehlendem Resolver, der nur umschifft werden kann, indem eine weitere Default-Handler-Methode hinzugefügt wird. Da dies nicht nötig ist, wenn nur die Nachricht übergeben wird, richt das ganze sehr nach einem Bug... --- diff --git a/src/main/java/de/juplo/kafka/ApplicationConfiguration.java b/src/main/java/de/juplo/kafka/ApplicationConfiguration.java index 51e7979..7f2bc49 100644 --- a/src/main/java/de/juplo/kafka/ApplicationConfiguration.java +++ b/src/main/java/de/juplo/kafka/ApplicationConfiguration.java @@ -12,12 +12,13 @@ import org.springframework.context.annotation.Configuration; import org.springframework.kafka.core.*; import org.springframework.kafka.listener.DeadLetterPublishingRecoverer; import org.springframework.kafka.listener.DefaultErrorHandler; +import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; import org.springframework.kafka.support.serializer.DelegatingByTypeSerializer; import org.springframework.kafka.support.serializer.JsonSerializer; import org.springframework.util.backoff.FixedBackOff; import java.util.Map; -import java.util.function.Consumer; +import java.util.function.BiConsumer; @Configuration @@ -26,20 +27,20 @@ import java.util.function.Consumer; public class ApplicationConfiguration { @Bean - public Consumer messageHandler() + public BiConsumer messageHandler() { - return (message) -> + return (message, metadata) -> { - log.info("Received ClientMessage: {}", message); + log.info("Received ClientMessage: {}, Metadata: {}", message, metadata); }; } @Bean - public Consumer greetingsHandler() + public BiConsumer greetingsHandler() { - return (greeting) -> + return (greeting, metadata) -> { - log.info("Received Greeting: {}", greeting); + log.info("Received Greeting: {}, Metadata: {}", greeting, metadata); }; } @@ -50,7 +51,8 @@ public class ApplicationConfiguration new StringSerializer(), new DelegatingByTypeSerializer(Map.of( byte[].class, new ByteArraySerializer(), - Object.class, new JsonSerializer<>()))); + ClientMessage.class, new JsonSerializer<>(), + Greeting.class, new JsonSerializer<>()))); } @Bean diff --git a/src/main/java/de/juplo/kafka/EndlessConsumer.java b/src/main/java/de/juplo/kafka/EndlessConsumer.java index 0a864da..cb2a3b9 100644 --- a/src/main/java/de/juplo/kafka/EndlessConsumer.java +++ b/src/main/java/de/juplo/kafka/EndlessConsumer.java @@ -10,12 +10,16 @@ import org.springframework.kafka.annotation.KafkaHandler; import org.springframework.kafka.annotation.KafkaListener; import org.springframework.kafka.config.KafkaListenerEndpointRegistry; import org.springframework.kafka.listener.ConsumerAwareRebalanceListener; +import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; +import org.springframework.kafka.support.KafkaHeaders; +import org.springframework.messaging.handler.annotation.Header; +import org.springframework.messaging.handler.annotation.Payload; import org.springframework.stereotype.Component; import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.function.Consumer; +import java.util.function.BiConsumer; @Component @@ -33,9 +37,9 @@ public class EndlessConsumer implements ConsumerAwareRebalanceListener @Value("${spring.kafka.consumer.client-id}") String id; @Autowired - Consumer messageHandler; + BiConsumer messageHandler; @Autowired - Consumer greetingsHandler; + BiConsumer greetingsHandler; private long consumed = 0; @@ -90,16 +94,28 @@ public class EndlessConsumer implements ConsumerAwareRebalanceListener @KafkaHandler - public void receiveGreeting(Greeting greeting) + public void receiveMessage( + @Payload ClientMessage message, + @Header(KafkaHeaders.RECORD_METADATA) ConsumerRecordMetadata metadata) { - greetingsHandler.accept(greeting); + messageHandler.accept(message, metadata); consumed++; } @KafkaHandler - public void receiveMessage(ClientMessage message) + public void receiveGreeting( + @Payload Greeting greeting, + @Header(KafkaHeaders.RECORD_METADATA) ConsumerRecordMetadata metadata) + { + greetingsHandler.accept(greeting, metadata); + consumed++; + } + + @KafkaHandler(isDefault = true) + public void unknown( + @Payload Object unknown, + @Header(KafkaHeaders.RECORD_METADATA) ConsumerRecordMetadata metadata) { - messageHandler.accept(message); consumed++; } diff --git a/src/test/java/de/juplo/kafka/ApplicationTests.java b/src/test/java/de/juplo/kafka/ApplicationTests.java index 8a485b3..f13fdde 100644 --- a/src/test/java/de/juplo/kafka/ApplicationTests.java +++ b/src/test/java/de/juplo/kafka/ApplicationTests.java @@ -1,7 +1,6 @@ package de.juplo.kafka; import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.producer.KafkaProducer; import org.apache.kafka.clients.producer.ProducerRecord; @@ -17,6 +16,7 @@ import org.springframework.boot.test.context.TestConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; +import org.springframework.kafka.listener.adapter.ConsumerRecordMetadata; import org.springframework.kafka.support.serializer.JsonSerializer; import org.springframework.kafka.test.context.EmbeddedKafka; import org.springframework.test.context.TestPropertySource; @@ -27,7 +27,6 @@ import java.util.*; import java.util.concurrent.ExecutionException; import java.util.function.BiConsumer; import java.util.function.BiFunction; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -71,11 +70,11 @@ class ApplicationTests @Autowired EndlessConsumer endlessConsumer; @Autowired - RecordHandler recordHandler; + ClientMessageHandler clientMessageHandler; Map oldOffsets; Map newOffsets; - Set> receivedRecords; + Set received; /** Tests methods */ @@ -87,7 +86,7 @@ class ApplicationTests await("100 records received") .atMost(Duration.ofSeconds(30)) - .until(() -> receivedRecords.size() == 100); + .until(() -> received.size() == 100); await("Offsets committed") .atMost(Duration.ofSeconds(10)) @@ -112,7 +111,7 @@ class ApplicationTests await("99 records received") .atMost(Duration.ofSeconds(30)) - .until(() -> receivedRecords.size() == 99); + .until(() -> received.size() == 99); await("Offsets committed") .atMost(Duration.ofSeconds(10)) @@ -137,17 +136,17 @@ class ApplicationTests @Test void commitsOffsetOnProgramLogicErrorFoo() { - recordHandler.testHandler = (record) -> + clientMessageHandler.testHandler = (clientMessage, metadata) -> { - if (Integer.parseInt(record.value().message)%10 ==0) - throw new RuntimeException("BOOM: " + record.value().message + "%10 == 0"); + if (Integer.parseInt(clientMessage.message)%10 ==0) + throw new RuntimeException("BOOM: " + clientMessage.message + "%10 == 0"); }; send100Messages((key, counter) -> serialize(key, counter)); await("80 records received") .atMost(Duration.ofSeconds(30)) - .until(() -> receivedRecords.size() == 100); + .until(() -> received.size() == 100); await("Offsets committed") .atMost(Duration.ofSeconds(10)) @@ -272,11 +271,11 @@ class ApplicationTests @BeforeEach public void init() { - recordHandler.testHandler = (record) -> {}; + clientMessageHandler.testHandler = (clientMessage, metadata) -> {}; oldOffsets = new HashMap<>(); newOffsets = new HashMap<>(); - receivedRecords = new HashSet<>(); + received = new HashSet<>(); doForCurrentOffsets((tp, offset) -> { @@ -284,14 +283,13 @@ class ApplicationTests newOffsets.put(tp, offset - 1); }); - recordHandler.captureOffsets = - record -> + clientMessageHandler.captureOffsets = + (clientMessage, metadata) -> { - receivedRecords.add(record); - log.debug("TEST: Processing record #{}: {}", receivedRecords.size(), record.value()); + received.add(clientMessage); + log.debug("TEST: Processing record #{}: {}", received.size(), clientMessage); newOffsets.put( - new TopicPartition(record.topic(), record.partition()), - record.offset()); + new TopicPartition(metadata.topic(), metadata.partition()), metadata.offset()); }; endlessConsumer.start(); @@ -310,18 +308,18 @@ class ApplicationTests } } - public static class RecordHandler implements Consumer> + public static class ClientMessageHandler implements BiConsumer { - Consumer> captureOffsets; - Consumer> testHandler; + BiConsumer captureOffsets; + BiConsumer testHandler; @Override - public void accept(ConsumerRecord record) + public void accept(ClientMessage clientMessage, ConsumerRecordMetadata metadata) { captureOffsets .andThen(testHandler) - .accept(record); + .accept(clientMessage, metadata); } } @@ -331,9 +329,9 @@ class ApplicationTests { @Primary @Bean - public Consumer> testHandler() + public BiConsumer testHandler() { - return new RecordHandler(); + return new ClientMessageHandler(); } @Bean