TMP

[demos/kafka/training] / src / test / java / de / juplo / kafka / ApplicationTests.java

diff --git a/src/test/java/de/juplo/kafka/ApplicationTests.java b/src/test/java/de/juplo/kafka/ApplicationTests.java
index ca72e3c..09614b8 100644 (file)
--- a/src/test/java/de/juplo/kafka/ApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/ApplicationTests.java
@@ -20,14 +20,11 @@ import org.springframework.kafka.test.context.EmbeddedKafka;
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
 
-import java.time.Clock;
 import java.time.Duration;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -64,6 +61,8 @@ class ApplicationTests
        @Autowired
        KafkaConsumer<String, String> kafkaConsumer;
        @Autowired
+       KafkaConsumer<Bytes, Bytes> offsetConsumer;
+       @Autowired
        PartitionStatisticsRepository partitionStatisticsRepository;
        @Autowired
        ApplicationProperties properties;
@@ -71,8 +70,11 @@ class ApplicationTests
        ExecutorService executor;
        @Autowired
        PartitionStatisticsRepository repository;
+       @Autowired
+       SumRebalanceListener sumRebalanceListener;
+       @Autowired
+       SumRecordHandler sumRecordHandler;
 
-       Consumer<ConsumerRecord<String, String>> testHandler;
        EndlessConsumer<String, String> endlessConsumer;
        Map<TopicPartition, Long> oldOffsets;
        Map<TopicPartition, Long> newOffsets;
@@ -84,14 +86,20 @@ class ApplicationTests
        @Test
        void commitsCurrentOffsetsOnSuccess() throws ExecutionException, InterruptedException
        {
-               send100Messages(i ->  new Bytes(valueSerializer.serialize(TOPIC, i)));
+               send100Messages((partition, key, counter) ->
+               {
+                       Bytes value = new Bytes(valueSerializer.serialize(TOPIC, counter));
+                       return new ProducerRecord<>(TOPIC, partition, key, value);
+               });
 
                await("100 records received")
                                .atMost(Duration.ofSeconds(30))
+                               .pollInterval(Duration.ofSeconds(1))
                                .until(() -> receivedRecords.size() >= 100);
 
                await("Offsets committed")
                                .atMost(Duration.ofSeconds(10))
+                               .pollInterval(Duration.ofSeconds(1))
                                .untilAsserted(() ->
                                {
                                        checkSeenOffsetsForProgress();
@@ -124,8 +132,8 @@ class ApplicationTests
                Set<TopicPartition> withProgress = new HashSet<>();
                partitions().forEach(tp ->
                {
-                       Long oldOffset = oldOffsets.get(tp);
-                       Long newOffset = newOffsets.get(tp);
+                       Long oldOffset = oldOffsets.get(tp) + 1;
+                       Long newOffset = newOffsets.get(tp) + 1;
                        if (!oldOffset.equals(newOffset))
                        {
                                log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -140,6 +148,24 @@ class ApplicationTests
 
        /** Helper methods for setting up and running the tests */
 
+       void seekToEnd()
+       {
+               offsetConsumer.assign(partitions());
+               partitions().forEach(tp ->
+               {
+                       Long offset = offsetConsumer.position(tp);
+                       log.info("New position for {}: {}", tp, offset);
+                       Integer partition = tp.partition();
+                       StatisticsDocument document =
+                                       partitionStatisticsRepository
+                                                       .findById(partition.toString())
+                                                       .orElse(new StatisticsDocument(partition));
+                       document.offset = offset;
+                       partitionStatisticsRepository.save(document);
+               });
+               offsetConsumer.unsubscribe();
+       }
+
        void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
        {
                partitions().forEach(tp ->
@@ -160,7 +186,12 @@ class ApplicationTests
        }
 
 
-       void send100Messages(Function<Long, Bytes> messageGenerator)
+       public interface RecordGenerator<K, V>
+       {
+               public ProducerRecord<String, Bytes> generate(int partition, String key, long counter);
+       }
+
+       void send100Messages(RecordGenerator recordGenerator)
        {
                long i = 0;
 
@@ -168,14 +199,8 @@ class ApplicationTests
                {
                        for (int key = 0; key < 10; key++)
                        {
-                               Bytes value = messageGenerator.apply(++i);
-
                                ProducerRecord<String, Bytes> record =
-                                               new ProducerRecord<>(
-                                                               TOPIC,
-                                                               partition,
-                                                               Integer.toString(key%2),
-                                                               value);
+                                               recordGenerator.generate(partition, Integer.toString(partition*10+key%2), ++i);
 
                                kafkaProducer.send(record, (metadata, e) ->
                                {
@@ -205,7 +230,7 @@ class ApplicationTests
        @BeforeEach
        public void init()
        {
-               testHandler = record -> {} ;
+               seekToEnd();
 
                oldOffsets = new HashMap<>();
                newOffsets = new HashMap<>();
@@ -217,25 +242,25 @@ class ApplicationTests
                        newOffsets.put(tp, offset - 1);
                });
 
-               Consumer<ConsumerRecord<String, String>> captureOffsetAndExecuteTestHandler =
-                               record ->
-                               {
-                                       newOffsets.put(
-                                                       new TopicPartition(record.topic(), record.partition()),
-                                                       record.offset());
-                                       receivedRecords.add(record);
-                                       testHandler.accept(record);
+               TestRecordHandler<String, String> captureOffsetAndExecuteTestHandler =
+                               new TestRecordHandler<String, String>(sumRecordHandler) {
+                                       @Override
+                                       public void onNewRecord(ConsumerRecord<String, String> record)
+                                       {
+                                               newOffsets.put(
+                                                               new TopicPartition(record.topic(), record.partition()),
+                                                               record.offset());
+                                               receivedRecords.add(record);
+                                       }
                                };
 
                endlessConsumer =
                                new EndlessConsumer<>(
                                                executor,
-                                               repository,
                                                properties.getClientId(),
                                                properties.getTopic(),
-                                               Clock.systemDefaultZone(),
-                                               properties.getCommitInterval(),
                                                kafkaConsumer,
+                                               sumRebalanceListener,
                                                captureOffsetAndExecuteTestHandler);
 
                endlessConsumer.start();
@@ -276,5 +301,19 @@ class ApplicationTests
 
                        return new KafkaProducer<>(props);
                }
+
+               @Bean
+               KafkaConsumer<Bytes, Bytes> offsetConsumer(ApplicationProperties properties)
+               {
+                       Properties props = new Properties();
+                       props.put("bootstrap.servers", properties.getBootstrapServer());
+                       props.put("client.id", "OFFSET-CONSUMER");
+                       props.put("enable.auto.commit", false);
+                       props.put("auto.offset.reset", "latest");
+                       props.put("key.deserializer", BytesDeserializer.class.getName());
+                       props.put("value.deserializer", BytesDeserializer.class.getName());
+
+                       return new KafkaConsumer<>(props);
+               }
        }
 }