X-Git-Url: http://juplo.de/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Fde%2Fjuplo%2Fkafka%2FGenericApplicationTests.java;h=595ef89d29001fbc39a34cf1d757e8ecadc7d69f;hb=2d25525ef70a90709edc48bd9542d1b08a2888a2;hp=b019373250d2c5c15062d43c4b262d5d7486f5e8;hpb=d2eb370acf1a2195c36421ffc471f67cb4a8e86e;p=demos%2Fkafka%2Ftraining

diff --git a/src/test/java/de/juplo/kafka/GenericApplicationTests.java b/src/test/java/de/juplo/kafka/GenericApplicationTests.java
index b019373..595ef89 100644
--- a/src/test/java/de/juplo/kafka/GenericApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/GenericApplicationTests.java
@@ -2,6 +2,7 @@ package de.juplo.kafka;
 
 import com.mongodb.client.MongoClient;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
 import org.apache.kafka.clients.consumer.ConsumerRecord;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
@@ -62,13 +63,11 @@ abstract class GenericApplicationTests<K, V>
 	@Autowired
 	ExecutorService executor;
 	@Autowired
-	StateRepository stateRepository;
-	@Autowired
 	MongoClient mongoClient;
 	@Autowired
 	MongoProperties mongoProperties;
 	@Autowired
-	PollIntervalAwareConsumerRebalanceListener rebalanceListener;
+	ConsumerRebalanceListener rebalanceListener;
 	@Autowired
 	RecordHandler<K, V> recordHandler;
 
@@ -76,7 +75,7 @@ abstract class GenericApplicationTests<K, V>
 	KafkaConsumer<Bytes, Bytes> offsetConsumer;
 	EndlessConsumer<K, V> endlessConsumer;
 	Map<TopicPartition, Long> oldOffsets;
-	Map<TopicPartition, Long> newOffsets;
+	Map<TopicPartition, Long> seenOffsets;
 	Set<ConsumerRecord<K, V>> receivedRecords;
 
 
@@ -93,7 +92,7 @@ abstract class GenericApplicationTests<K, V>
 	/** Tests methods */
 
 	@Test
-	void commitsCurrentOffsetsOnSuccess()
+	void commitsCurrentOffsetsOnSuccess() throws Exception
 	{
 		int numberOfGeneratedMessages =
 				recordGenerator.generate(false, false, messageSender);
@@ -109,13 +108,14 @@ abstract class GenericApplicationTests<K, V>
 				.untilAsserted(() ->
 				{
 					checkSeenOffsetsForProgress();
-					compareToCommitedOffsets(newOffsets);
+					assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 				});
 
 		assertThatExceptionOfType(IllegalStateException.class)
 				.isThrownBy(() -> endlessConsumer.exitStatus())
 				.describedAs("Consumer should still be running");
 
+		endlessConsumer.stop();
 		recordGenerator.assertBusinessLogic();
 	}
 
@@ -132,7 +132,7 @@ abstract class GenericApplicationTests<K, V>
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(newOffsets);
+		assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 
 		endlessConsumer.start();
 		await("Consumer failed")
@@ -141,7 +141,7 @@ abstract class GenericApplicationTests<K, V>
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(newOffsets);
+		assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
 		assertThat(receivedRecords.size())
 				.describedAs("Received not all sent events")
 				.isLessThan(numberOfGeneratedMessages);
@@ -169,7 +169,7 @@ abstract class GenericApplicationTests<K, V>
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(oldOffsets);
+		assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
 
 		endlessConsumer.start();
 		await("Consumer failed")
@@ -177,8 +177,7 @@ abstract class GenericApplicationTests<K, V>
 				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> !endlessConsumer.running());
 
-		checkSeenOffsetsForProgress();
-		compareToCommitedOffsets(oldOffsets);
+		assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
 
 		assertThatNoException()
 				.describedAs("Consumer should not be running")
@@ -193,18 +192,37 @@ abstract class GenericApplicationTests<K, V>
 
 	/** Helper methods for the verification of expectations */
 
-	void compareToCommitedOffsets(Map<TopicPartition, Long> offsetsToCheck)
+	void assertSeenOffsetsEqualCommittedOffsets(Map<TopicPartition, Long> offsetsToCheck)
 	{
 		doForCurrentOffsets((tp, offset) ->
 		{
 			Long expected = offsetsToCheck.get(tp) + 1;
-			log.debug("Checking, if the offset for {} is {}", tp, expected);
+			log.debug("Checking, if the offset {} for {} is exactly {}", offset, tp, expected);
 			assertThat(offset)
 					.describedAs("Committed offset corresponds to the offset of the consumer")
 					.isEqualTo(expected);
 		});
 	}
 
+	void assertSeenOffsetsAreBehindCommittedOffsets(Map<TopicPartition, Long> offsetsToCheck)
+	{
+		List<Boolean> isOffsetBehindSeen = new LinkedList<>();
+
+		doForCurrentOffsets((tp, offset) ->
+		{
+			Long expected = offsetsToCheck.get(tp) + 1;
+			log.debug("Checking, if the offset {} for {} is at most {}", offset, tp, expected);
+			assertThat(offset)
+					.describedAs("Committed offset corresponds to the offset of the consumer")
+					.isLessThanOrEqualTo(expected);
+			isOffsetBehindSeen.add(offset < expected);
+		});
+
+		assertThat(isOffsetBehindSeen.stream().reduce(false, (result, next) -> result | next))
+				.describedAs("Committed offsets are behind seen offsets")
+				.isTrue();
+	}
+
 	void checkSeenOffsetsForProgress()
 	{
 		// Be sure, that some messages were consumed...!
@@ -212,7 +230,7 @@ abstract class GenericApplicationTests<K, V>
 		partitions().forEach(tp ->
 		{
 			Long oldOffset = oldOffsets.get(tp) + 1;
-			Long newOffset = newOffsets.get(tp) + 1;
+			Long newOffset = seenOffsets.get(tp) + 1;
 			if (!oldOffset.equals(newOffset))
 			{
 				log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -230,29 +248,23 @@ abstract class GenericApplicationTests<K, V>
 	void seekToEnd()
 	{
 		offsetConsumer.assign(partitions());
+		offsetConsumer.seekToEnd(partitions());
 		partitions().forEach(tp ->
 		{
+			// seekToEnd() works lazily: it only takes effect on poll()/position()
 			Long offset = offsetConsumer.position(tp);
 			log.info("New position for {}: {}", tp, offset);
-			Integer partition = tp.partition();
-			StateDocument document =
-					stateRepository
-							.findById(partition.toString())
-							.orElse(new StateDocument(partition));
-			document.offset = offset;
-			stateRepository.save(document);
 		});
+		// The new positions must be commited!
+		offsetConsumer.commitSync();
 		offsetConsumer.unsubscribe();
 	}
 
 	void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
 	{
-		partitions().forEach(tp ->
-		{
-			String partition = Integer.toString(tp.partition());
-			Optional<Long> offset = stateRepository.findById(partition).map(document -> document.offset);
-			consumer.accept(tp, offset.orElse(0l));
-		});
+		offsetConsumer.assign(partitions());
+		partitions().forEach(tp -> consumer.accept(tp, offsetConsumer.position(tp)));
+		offsetConsumer.unsubscribe();
 	}
 
 	List<TopicPartition> partitions()
@@ -336,13 +348,13 @@ abstract class GenericApplicationTests<K, V>
 		seekToEnd();
 
 		oldOffsets = new HashMap<>();
-		newOffsets = new HashMap<>();
+		seenOffsets = new HashMap<>();
 		receivedRecords = new HashSet<>();
 
 		doForCurrentOffsets((tp, offset) ->
 		{
 			oldOffsets.put(tp, offset - 1);
-			newOffsets.put(tp, offset - 1);
+			seenOffsets.put(tp, offset - 1);
 		});
 
 		TestRecordHandler<K, V> captureOffsetAndExecuteTestHandler =
@@ -351,7 +363,7 @@ abstract class GenericApplicationTests<K, V>
 					@Override
 					public void onNewRecord(ConsumerRecord<K, V> record)
 					{
-						newOffsets.put(
+						seenOffsets.put(
 								new TopicPartition(record.topic(), record.partition()),
 								record.offset());
 						receivedRecords.add(record);
@@ -375,7 +387,6 @@ abstract class GenericApplicationTests<K, V>
 	{
 		try
 		{
-			endlessConsumer.stop();
 			testRecordProducer.close();
 			offsetConsumer.close();
 		}