GRÜN: Korrektur der falsch formulierten Erwartungen zu dem Consumer-Zustand

[demos/kafka/training] / src / test / java / de / juplo / kafka / GenericApplicationTests.java

diff --git a/src/test/java/de/juplo/kafka/GenericApplicationTests.java b/src/test/java/de/juplo/kafka/GenericApplicationTests.java
index 93daf6b..0f40058 100644 (file)
--- a/src/test/java/de/juplo/kafka/GenericApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/GenericApplicationTests.java
@@ -62,6 +62,8 @@ abstract class GenericApplicationTests<K, V>
        @Autowired
        ExecutorService executor;
        @Autowired
+       StateRepository stateRepository;
+       @Autowired
        MongoClient mongoClient;
        @Autowired
        MongoProperties mongoProperties;
@@ -74,7 +76,7 @@ abstract class GenericApplicationTests<K, V>
        KafkaConsumer<Bytes, Bytes> offsetConsumer;
        EndlessConsumer<K, V> endlessConsumer;
        Map<TopicPartition, Long> oldOffsets;
-       Map<TopicPartition, Long> newOffsets;
+       Map<TopicPartition, Long> seenOffsets;
        Set<ConsumerRecord<K, V>> receivedRecords;
 
 
@@ -107,7 +109,7 @@ abstract class GenericApplicationTests<K, V>
                                .untilAsserted(() ->
                                {
                                        checkSeenOffsetsForProgress();
-                                       compareToCommitedOffsets(newOffsets);
+                                       assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
                                });
 
                assertThatExceptionOfType(IllegalStateException.class)
@@ -130,7 +132,7 @@ abstract class GenericApplicationTests<K, V>
                                .until(() -> !endlessConsumer.running());
 
                checkSeenOffsetsForProgress();
-               compareToCommitedOffsets(newOffsets);
+               assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 
                endlessConsumer.start();
                await("Consumer failed")
@@ -139,7 +141,7 @@ abstract class GenericApplicationTests<K, V>
                                .until(() -> !endlessConsumer.running());
 
                checkSeenOffsetsForProgress();
-               compareToCommitedOffsets(newOffsets);
+               assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
                assertThat(receivedRecords.size())
                                .describedAs("Received not all sent events")
                                .isLessThan(numberOfGeneratedMessages);
@@ -167,7 +169,7 @@ abstract class GenericApplicationTests<K, V>
                                .until(() -> !endlessConsumer.running());
 
                checkSeenOffsetsForProgress();
-               compareToCommitedOffsets(oldOffsets);
+               assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
 
                endlessConsumer.start();
                await("Consumer failed")
@@ -175,8 +177,7 @@ abstract class GenericApplicationTests<K, V>
                                .pollInterval(Duration.ofSeconds(1))
                                .until(() -> !endlessConsumer.running());
 
-               checkSeenOffsetsForProgress();
-               compareToCommitedOffsets(oldOffsets);
+               assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
 
                assertThatNoException()
                                .describedAs("Consumer should not be running")
@@ -191,18 +192,37 @@ abstract class GenericApplicationTests<K, V>
 
        /** Helper methods for the verification of expectations */
 
-       void compareToCommitedOffsets(Map<TopicPartition, Long> offsetsToCheck)
+       void assertSeenOffsetsEqualCommittedOffsets(Map<TopicPartition, Long> offsetsToCheck)
        {
                doForCurrentOffsets((tp, offset) ->
                {
                        Long expected = offsetsToCheck.get(tp) + 1;
-                       log.debug("Checking, if the offset for {} is {}", tp, expected);
+                       log.debug("Checking, if the offset {} for {} is exactly {}", offset, tp, expected);
                        assertThat(offset)
                                        .describedAs("Committed offset corresponds to the offset of the consumer")
                                        .isEqualTo(expected);
                });
        }
 
+       void assertSeenOffsetsAreBehindCommittedOffsets(Map<TopicPartition, Long> offsetsToCheck)
+       {
+               List<Boolean> isOffsetBehindSeen = new LinkedList<>();
+
+               doForCurrentOffsets((tp, offset) ->
+               {
+                       Long expected = offsetsToCheck.get(tp) + 1;
+                       log.debug("Checking, if the offset {} for {} is at most {}", offset, tp, expected);
+                       assertThat(offset)
+                                       .describedAs("Committed offset corresponds to the offset of the consumer")
+                                       .isLessThanOrEqualTo(expected);
+                       isOffsetBehindSeen.add(offset < expected);
+               });
+
+               assertThat(isOffsetBehindSeen.stream().reduce(false, (result, next) -> result | next))
+                               .describedAs("Committed offsets are behind seen offsets")
+                               .isTrue();
+       }
+
        void checkSeenOffsetsForProgress()
        {
                // Be sure, that some messages were consumed...!
@@ -210,7 +230,7 @@ abstract class GenericApplicationTests<K, V>
                partitions().forEach(tp ->
                {
                        Long oldOffset = oldOffsets.get(tp) + 1;
-                       Long newOffset = newOffsets.get(tp) + 1;
+                       Long newOffset = seenOffsets.get(tp) + 1;
                        if (!oldOffset.equals(newOffset))
                        {
                                log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -228,23 +248,29 @@ abstract class GenericApplicationTests<K, V>
        void seekToEnd()
        {
                offsetConsumer.assign(partitions());
-               offsetConsumer.seekToEnd(partitions());
                partitions().forEach(tp ->
                {
-                       // seekToEnd() works lazily: it only takes effect on poll()/position()
                        Long offset = offsetConsumer.position(tp);
                        log.info("New position for {}: {}", tp, offset);
+                       Integer partition = tp.partition();
+                       StateDocument document =
+                                       stateRepository
+                                                       .findById(partition.toString())
+                                                       .orElse(new StateDocument(partition));
+                       document.offset = offset;
+                       stateRepository.save(document);
                });
-               // The new positions must be commited!
-               offsetConsumer.commitSync();
                offsetConsumer.unsubscribe();
        }
 
        void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
        {
-               offsetConsumer.assign(partitions());
-               partitions().forEach(tp -> consumer.accept(tp, offsetConsumer.position(tp)));
-               offsetConsumer.unsubscribe();
+               partitions().forEach(tp ->
+               {
+                       String partition = Integer.toString(tp.partition());
+                       Optional<Long> offset = stateRepository.findById(partition).map(document -> document.offset);
+                       consumer.accept(tp, offset.orElse(0l));
+               });
        }
 
        List<TopicPartition> partitions()
@@ -324,16 +350,17 @@ abstract class GenericApplicationTests<K, V>
                props.put("value.deserializer", BytesDeserializer.class.getName());
                offsetConsumer = new KafkaConsumer<>(props);
 
+               mongoClient.getDatabase(mongoProperties.getDatabase()).drop();
                seekToEnd();
 
                oldOffsets = new HashMap<>();
-               newOffsets = new HashMap<>();
+               seenOffsets = new HashMap<>();
                receivedRecords = new HashSet<>();
 
                doForCurrentOffsets((tp, offset) ->
                {
                        oldOffsets.put(tp, offset - 1);
-                       newOffsets.put(tp, offset - 1);
+                       seenOffsets.put(tp, offset - 1);
                });
 
                TestRecordHandler<K, V> captureOffsetAndExecuteTestHandler =
@@ -342,15 +369,13 @@ abstract class GenericApplicationTests<K, V>
                                        @Override
                                        public void onNewRecord(ConsumerRecord<K, V> record)
                                        {
-                                               newOffsets.put(
+                                               seenOffsets.put(
                                                                new TopicPartition(record.topic(), record.partition()),
                                                                record.offset());
                                                receivedRecords.add(record);
                                        }
                                };
 
-               mongoClient.getDatabase(mongoProperties.getDatabase()).drop();
-
                endlessConsumer =
                                new EndlessConsumer<>(
                                                executor,