X-Git-Url: https://juplo.de/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Fde%2Fjuplo%2Fkafka%2FApplicationTests.java;h=fc5d4c9e51d21348a9271fad7877bbd07b9ed0fe;hb=2da45caa1f9d32e3a5506d71cce7f06fa2e36523;hp=39ff0d7110cac758547928f8dc77e1fceec50e25;hpb=221f774959bb0d1e113bdeefd8fdc52ea1291081;p=demos%2Fkafka%2Ftraining

diff --git a/src/test/java/de/juplo/kafka/ApplicationTests.java b/src/test/java/de/juplo/kafka/ApplicationTests.java
index 39ff0d7..fc5d4c9 100644
--- a/src/test/java/de/juplo/kafka/ApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/ApplicationTests.java
@@ -11,6 +11,8 @@ import org.apache.kafka.common.serialization.*;
 import org.apache.kafka.common.utils.Bytes;
 import org.junit.jupiter.api.*;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo;
 import org.springframework.boot.test.context.ConfigDataApplicationContextInitializer;
 import org.springframework.boot.test.context.TestConfiguration;
 import org.springframework.context.annotation.Bean;
@@ -24,8 +26,6 @@ import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -40,8 +40,12 @@ import static org.awaitility.Awaitility.*;
 @TestPropertySource(
 		properties = {
 				"consumer.bootstrap-server=${spring.embedded.kafka.brokers}",
-				"consumer.topic=" + TOPIC })
+				"consumer.topic=" + TOPIC,
+				"consumer.commit-interval=1s",
+				"spring.mongodb.embedded.version=4.4.13" })
 @EmbeddedKafka(topics = TOPIC, partitions = PARTITIONS)
+@EnableAutoConfiguration
+@AutoConfigureDataMongo
 @Slf4j
 class ApplicationTests
 {
@@ -60,11 +64,18 @@ class ApplicationTests
 	@Autowired
 	KafkaConsumer<Bytes, Bytes> offsetConsumer;
 	@Autowired
+	PartitionStatisticsRepository partitionStatisticsRepository;
+	@Autowired
 	ApplicationProperties properties;
 	@Autowired
 	ExecutorService executor;
+	@Autowired
+	PartitionStatisticsRepository repository;
+	@Autowired
+	KeyCountingRebalanceListener keyCountingRebalanceListener;
+	@Autowired
+	KeyCountingRecordHandler keyCountingRecordHandler;
 
-	Consumer<ConsumerRecord<String, Long>> testHandler;
 	EndlessConsumer<String, Long> endlessConsumer;
 	Map<TopicPartition, Long> oldOffsets;
 	Map<TopicPartition, Long> newOffsets;
@@ -74,7 +85,6 @@ class ApplicationTests
 	/** Tests methods */
 
 	@Test
-	@Order(1) // << The poistion pill is not skipped. Hence, this test must run first
 	void commitsCurrentOffsetsOnSuccess() throws ExecutionException, InterruptedException
 	{
 		send100Messages((partition, key, counter) ->
@@ -85,10 +95,12 @@ class ApplicationTests
 
 		await("100 records received")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> receivedRecords.size() >= 100);
 
 		await("Offsets committed")
 				.atMost(Duration.ofSeconds(10))
+				.pollInterval(Duration.ofSeconds(1))
 				.untilAsserted(() ->
 				{
 					checkSeenOffsetsForProgress();
@@ -101,8 +113,7 @@ class ApplicationTests
 	}
 
 	@Test
-	@Order(2)
-	void commitsOffsetOfErrorForReprocessingOnError()
+	void commitsOffsetOfErrorForReprocessingOnDeserializationError()
 	{
 		send100Messages((partition, key, counter) ->
 		{
@@ -114,6 +125,7 @@ class ApplicationTests
 
 		await("Consumer failed")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
@@ -122,6 +134,7 @@ class ApplicationTests
 		endlessConsumer.start();
 		await("Consumer failed")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
@@ -159,8 +172,8 @@ class ApplicationTests
 		Set<TopicPartition> withProgress = new HashSet<>();
 		partitions().forEach(tp ->
 		{
-			Long oldOffset = oldOffsets.get(tp);
-			Long newOffset = newOffsets.get(tp);
+			Long oldOffset = oldOffsets.get(tp) + 1;
+			Long newOffset = newOffsets.get(tp) + 1;
 			if (!oldOffset.equals(newOffset))
 			{
 				log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -175,13 +188,34 @@ class ApplicationTests
 
 	/** Helper methods for setting up and running the tests */
 
-	void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
+	void seekToEnd()
 	{
 		offsetConsumer.assign(partitions());
-		partitions().forEach(tp -> consumer.accept(tp, offsetConsumer.position(tp)));
+		partitions().forEach(tp ->
+		{
+			Long offset = offsetConsumer.position(tp);
+			log.info("New position for {}: {}", tp, offset);
+			Integer partition = tp.partition();
+			StatisticsDocument document =
+					partitionStatisticsRepository
+							.findById(partition.toString())
+							.orElse(new StatisticsDocument(partition));
+			document.offset = offset;
+			partitionStatisticsRepository.save(document);
+		});
 		offsetConsumer.unsubscribe();
 	}
 
+	void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
+	{
+		partitions().forEach(tp ->
+		{
+			String partition = Integer.toString(tp.partition());
+			Optional<Long> offset = partitionStatisticsRepository.findById(partition).map(document -> document.offset);
+			consumer.accept(tp, offset.orElse(0l));
+		});
+		}
+
 	List<TopicPartition> partitions()
 	{
 		return
@@ -206,7 +240,7 @@ class ApplicationTests
 			for (int key = 0; key < 10; key++)
 			{
 				ProducerRecord<String, Bytes> record =
-						recordGenerator.generate(partition, Integer.toString(key%2), ++i);
+						recordGenerator.generate(partition, Integer.toString(partition*10+key%2), ++i);
 
 				kafkaProducer.send(record, (metadata, e) ->
 				{
@@ -236,7 +270,7 @@ class ApplicationTests
 	@BeforeEach
 	public void init()
 	{
-		testHandler = record -> {} ;
+		seekToEnd();
 
 		oldOffsets = new HashMap<>();
 		newOffsets = new HashMap<>();
@@ -248,14 +282,16 @@ class ApplicationTests
 			newOffsets.put(tp, offset - 1);
 		});
 
-		Consumer<ConsumerRecord<String, Long>> captureOffsetAndExecuteTestHandler =
-				record ->
-				{
-					newOffsets.put(
-							new TopicPartition(record.topic(), record.partition()),
-							record.offset());
-					receivedRecords.add(record);
-					testHandler.accept(record);
+		TestRecordHandler<String, Long> captureOffsetAndExecuteTestHandler =
+				new TestRecordHandler<String, Long>(keyCountingRecordHandler) {
+					@Override
+					public void onNewRecord(ConsumerRecord<String, Long> record)
+					{
+						newOffsets.put(
+								new TopicPartition(record.topic(), record.partition()),
+								record.offset());
+						receivedRecords.add(record);
+					}
 				};
 
 		endlessConsumer =
@@ -264,6 +300,7 @@ class ApplicationTests
 						properties.getClientId(),
 						properties.getTopic(),
 						kafkaConsumer,
+						keyCountingRebalanceListener,
 						captureOffsetAndExecuteTestHandler);
 
 		endlessConsumer.start();
@@ -311,7 +348,8 @@ class ApplicationTests
 			Properties props = new Properties();
 			props.put("bootstrap.servers", properties.getBootstrapServer());
 			props.put("client.id", "OFFSET-CONSUMER");
-			props.put("group.id", properties.getGroupId());
+			props.put("enable.auto.commit", false);
+			props.put("auto.offset.reset", "latest");
 			props.put("key.deserializer", BytesDeserializer.class.getName());
 			props.put("value.deserializer", BytesDeserializer.class.getName());