X-Git-Url: http://juplo.de/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Fde%2Fjuplo%2Fkafka%2FGenericApplicationTests.java;h=595ef89d29001fbc39a34cf1d757e8ecadc7d69f;hb=2d25525ef70a90709edc48bd9542d1b08a2888a2;hp=cff44e2581bfb016c31acccfaf4a9e597d9bc68c;hpb=de4f94c45fd0678777fecba4dbcb63f89e0ebafa;p=demos%2Fkafka%2Ftraining

diff --git a/src/test/java/de/juplo/kafka/GenericApplicationTests.java b/src/test/java/de/juplo/kafka/GenericApplicationTests.java
index cff44e2..595ef89 100644
--- a/src/test/java/de/juplo/kafka/GenericApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/GenericApplicationTests.java
@@ -1,6 +1,8 @@
 package de.juplo.kafka;
 
+import com.mongodb.client.MongoClient;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -12,6 +14,7 @@ import org.apache.kafka.common.utils.Bytes;
 import org.junit.jupiter.api.*;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.autoconfigure.mongo.MongoProperties;
 import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo;
 import org.springframework.boot.test.context.ConfigDataApplicationContextInitializer;
 import org.springframework.boot.test.context.TestConfiguration;
@@ -39,7 +42,7 @@ import static org.awaitility.Awaitility.*;
 		properties = {
 				"sumup.adder.bootstrap-server=${spring.embedded.kafka.brokers}",
 				"sumup.adder.topic=" + TOPIC,
-				"sumup.adder.commit-interval=1s",
+				"sumup.adder.commit-interval=500ms",
 				"spring.mongodb.embedded.version=4.4.13" })
 @EmbeddedKafka(topics = TOPIC, partitions = PARTITIONS)
 @EnableAutoConfiguration
@@ -60,7 +63,11 @@ abstract class GenericApplicationTests<K, V>
 	@Autowired
 	ExecutorService executor;
 	@Autowired
-	PollIntervalAwareConsumerRebalanceListener rebalanceListener;
+	MongoClient mongoClient;
+	@Autowired
+	MongoProperties mongoProperties;
+	@Autowired
+	ConsumerRebalanceListener rebalanceListener;
 	@Autowired
 	RecordHandler<K, V> recordHandler;
 
@@ -68,7 +75,7 @@ abstract class GenericApplicationTests<K, V>
 	KafkaConsumer<Bytes, Bytes> offsetConsumer;
 	EndlessConsumer<K, V> endlessConsumer;
 	Map<TopicPartition, Long> oldOffsets;
-	Map<TopicPartition, Long> newOffsets;
+	Map<TopicPartition, Long> seenOffsets;
 	Set<ConsumerRecord<K, V>> receivedRecords;
 
 
@@ -85,7 +92,7 @@ abstract class GenericApplicationTests<K, V>
 	/** Tests methods */
 
 	@Test
-	void commitsCurrentOffsetsOnSuccess()
+	void commitsCurrentOffsetsOnSuccess() throws Exception
 	{
 		int numberOfGeneratedMessages =
 				recordGenerator.generate(false, false, messageSender);
@@ -101,13 +108,14 @@ abstract class GenericApplicationTests<K, V>
 				.untilAsserted(() ->
 				{
 					checkSeenOffsetsForProgress();
-					compareToCommitedOffsets(newOffsets);
+					assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 				});
 
 		assertThatExceptionOfType(IllegalStateException.class)
 				.isThrownBy(() -> endlessConsumer.exitStatus())
 				.describedAs("Consumer should still be running");
 
+		endlessConsumer.stop();
 		recordGenerator.assertBusinessLogic();
 	}
 
@@ -124,7 +132,7 @@ abstract class GenericApplicationTests<K, V>
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(newOffsets);
+		assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 
 		endlessConsumer.start();
 		await("Consumer failed")
@@ -133,7 +141,7 @@ abstract class GenericApplicationTests<K, V>
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(newOffsets);
+		assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 		assertThat(receivedRecords.size())
 				.describedAs("Received not all sent events")
 				.isLessThan(numberOfGeneratedMessages);
@@ -161,7 +169,7 @@ abstract class GenericApplicationTests<K, V>
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(oldOffsets);
+		assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
 
 		endlessConsumer.start();
 		await("Consumer failed")
@@ -169,11 +177,7 @@ abstract class GenericApplicationTests<K, V>
 				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> !endlessConsumer.running());
 
-		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(oldOffsets);
-		assertThat(receivedRecords.size())
-				.describedAs("Received not all sent events")
-				.isLessThan(numberOfGeneratedMessages);
+		assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
 
 		assertThatNoException()
 				.describedAs("Consumer should not be running")
@@ -188,18 +192,37 @@ abstract class GenericApplicationTests<K, V>
 
 	/** Helper methods for the verification of expectations */
 
-	void compareToCommitedOffsets(Map<TopicPartition, Long> offsetsToCheck)
+	void assertSeenOffsetsEqualCommittedOffsets(Map<TopicPartition, Long> offsetsToCheck)
 	{
 		doForCurrentOffsets((tp, offset) ->
 		{
 			Long expected = offsetsToCheck.get(tp) + 1;
-			log.debug("Checking, if the offset for {} is {}", tp, expected);
+			log.debug("Checking, if the offset {} for {} is exactly {}", offset, tp, expected);
 			assertThat(offset)
 					.describedAs("Committed offset corresponds to the offset of the consumer")
 					.isEqualTo(expected);
 		});
 	}
 
+	void assertSeenOffsetsAreBehindCommittedOffsets(Map<TopicPartition, Long> offsetsToCheck)
+	{
+		List<Boolean> isOffsetBehindSeen = new LinkedList<>();
+
+		doForCurrentOffsets((tp, offset) ->
+		{
+			Long expected = offsetsToCheck.get(tp) + 1;
+			log.debug("Checking, if the offset {} for {} is at most {}", offset, tp, expected);
+			assertThat(offset)
+					.describedAs("Committed offset corresponds to the offset of the consumer")
+					.isLessThanOrEqualTo(expected);
+			isOffsetBehindSeen.add(offset < expected);
+		});
+
+		assertThat(isOffsetBehindSeen.stream().reduce(false, (result, next) -> result | next))
+				.describedAs("Committed offsets are behind seen offsets")
+				.isTrue();
+	}
+
 	void checkSeenOffsetsForProgress()
 	{
 		// Be sure, that some messages were consumed...!
@@ -207,7 +230,7 @@ abstract class GenericApplicationTests<K, V>
 		partitions().forEach(tp ->
 		{
 			Long oldOffset = oldOffsets.get(tp) + 1;
-			Long newOffset = newOffsets.get(tp) + 1;
+			Long newOffset = seenOffsets.get(tp) + 1;
 			if (!oldOffset.equals(newOffset))
 			{
 				log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -321,16 +344,17 @@ abstract class GenericApplicationTests<K, V>
 		props.put("value.deserializer", BytesDeserializer.class.getName());
 		offsetConsumer = new KafkaConsumer<>(props);
 
+		mongoClient.getDatabase(mongoProperties.getDatabase()).drop();
 		seekToEnd();
 
 		oldOffsets = new HashMap<>();
-		newOffsets = new HashMap<>();
+		seenOffsets = new HashMap<>();
 		receivedRecords = new HashSet<>();
 
 		doForCurrentOffsets((tp, offset) ->
 		{
 			oldOffsets.put(tp, offset - 1);
-			newOffsets.put(tp, offset - 1);
+			seenOffsets.put(tp, offset - 1);
 		});
 
 		TestRecordHandler<K, V> captureOffsetAndExecuteTestHandler =
@@ -339,7 +363,7 @@ abstract class GenericApplicationTests<K, V>
 					@Override
 					public void onNewRecord(ConsumerRecord<K, V> record)
 					{
-						newOffsets.put(
+						seenOffsets.put(
 								new TopicPartition(record.topic(), record.partition()),
 								record.offset());
 						receivedRecords.add(record);
@@ -363,7 +387,6 @@ abstract class GenericApplicationTests<K, V>
 	{
 		try
 		{
-			endlessConsumer.stop();
 			testRecordProducer.close();
 			offsetConsumer.close();
 		}