X-Git-Url: http://juplo.de/gitweb/?a=blobdiff_plain;f=src%2Ftest%2Fjava%2Fde%2Fjuplo%2Fkafka%2FApplicationTests.java;h=7f666f69b5412c242164b97c4c08ec6dae59008e;hb=cfbe7dcd3318ee846cb5890eea4328e36c5aa364;hp=a632a89c41f936b88f1f23742e78183279138ef3;hpb=b3777fba0ae679d9e2c9d36626fa208a952f83e8;p=demos%2Fkafka%2Ftraining

diff --git a/src/test/java/de/juplo/kafka/ApplicationTests.java b/src/test/java/de/juplo/kafka/ApplicationTests.java
index a632a89..7f666f6 100644
--- a/src/test/java/de/juplo/kafka/ApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/ApplicationTests.java
@@ -26,7 +26,6 @@ import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiConsumer;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -63,14 +62,12 @@ class ApplicationTests
 	@Autowired
 	KafkaConsumer<String, Long> kafkaConsumer;
 	@Autowired
-	PartitionStatisticsRepository partitionStatisticsRepository;
+	KafkaConsumer<Bytes, Bytes> offsetConsumer;
 	@Autowired
 	ApplicationProperties properties;
 	@Autowired
 	ExecutorService executor;
 	@Autowired
-	PartitionStatisticsRepository repository;
-	@Autowired
 	KeyCountingRebalanceListener keyCountingRebalanceListener;
 	@Autowired
 	KeyCountingRecordHandler keyCountingRecordHandler;
@@ -84,17 +81,22 @@ class ApplicationTests
 	/** Tests methods */
 
 	@Test
-	@Order(1) // << The poistion pill is not skipped. Hence, this test must run first
 	void commitsCurrentOffsetsOnSuccess() throws ExecutionException, InterruptedException
 	{
-		send100Messages(i ->  new Bytes(valueSerializer.serialize(TOPIC, i)));
+		send100Messages((partition, key, counter) ->
+		{
+			Bytes value = new Bytes(valueSerializer.serialize(TOPIC, counter));
+			return new ProducerRecord<>(TOPIC, partition, key, value);
+		});
 
 		await("100 records received")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> receivedRecords.size() >= 100);
 
 		await("Offsets committed")
 				.atMost(Duration.ofSeconds(10))
+				.pollInterval(Duration.ofSeconds(1))
 				.untilAsserted(() ->
 				{
 					checkSeenOffsetsForProgress();
@@ -107,16 +109,19 @@ class ApplicationTests
 	}
 
 	@Test
-	@Order(2)
-	void commitsOffsetOfErrorForReprocessingOnError()
+	void commitsOffsetOfErrorForReprocessingOnDeserializationError()
 	{
-		send100Messages(counter ->
-				counter == 77
-						? new Bytes(stringSerializer.serialize(TOPIC, "BOOM!"))
-						: new Bytes(valueSerializer.serialize(TOPIC, counter)));
+		send100Messages((partition, key, counter) ->
+		{
+			Bytes value = counter == 77
+					? new Bytes(stringSerializer.serialize(TOPIC, "BOOM!"))
+					: new Bytes(valueSerializer.serialize(TOPIC, counter));
+			return new ProducerRecord<>(TOPIC, partition, key, value);
+		});
 
 		await("Consumer failed")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
@@ -125,6 +130,7 @@ class ApplicationTests
 		endlessConsumer.start();
 		await("Consumer failed")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> !endlessConsumer.running());
 
 		checkSeenOffsetsForProgress();
@@ -162,8 +168,8 @@ class ApplicationTests
 		Set<TopicPartition> withProgress = new HashSet<>();
 		partitions().forEach(tp ->
 		{
-			Long oldOffset = oldOffsets.get(tp);
-			Long newOffset = newOffsets.get(tp);
+			Long oldOffset = oldOffsets.get(tp) + 1;
+			Long newOffset = newOffsets.get(tp) + 1;
 			if (!oldOffset.equals(newOffset))
 			{
 				log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -178,15 +184,27 @@ class ApplicationTests
 
 	/** Helper methods for setting up and running the tests */
 
-	void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
+	void seekToEnd()
 	{
+		offsetConsumer.assign(partitions());
+		offsetConsumer.seekToEnd(partitions());
 		partitions().forEach(tp ->
 		{
-			String partition = Integer.toString(tp.partition());
-			Optional<Long> offset = partitionStatisticsRepository.findById(partition).map(document -> document.offset);
-			consumer.accept(tp, offset.orElse(0l));
+			// seekToEnd() works lazily: it only takes effect on poll()/position()
+			Long offset = offsetConsumer.position(tp);
+			log.info("New position for {}: {}", tp, offset);
 		});
-		}
+		// The new positions must be commited!
+		offsetConsumer.commitSync();
+		offsetConsumer.unsubscribe();
+	}
+
+	void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
+	{
+		offsetConsumer.assign(partitions());
+		partitions().forEach(tp -> consumer.accept(tp, offsetConsumer.position(tp)));
+		offsetConsumer.unsubscribe();
+	}
 
 	List<TopicPartition> partitions()
 	{
@@ -198,7 +216,12 @@ class ApplicationTests
 	}
 
 
-	void send100Messages(Function<Long, Bytes> messageGenerator)
+	public interface RecordGenerator<K, V>
+	{
+		public ProducerRecord<String, Bytes> generate(int partition, String key, long counter);
+	}
+
+	void send100Messages(RecordGenerator recordGenerator)
 	{
 		long i = 0;
 
@@ -206,14 +229,8 @@ class ApplicationTests
 		{
 			for (int key = 0; key < 10; key++)
 			{
-				Bytes value = messageGenerator.apply(++i);
-
 				ProducerRecord<String, Bytes> record =
-						new ProducerRecord<>(
-								TOPIC,
-								partition,
-								Integer.toString(key%2),
-								value);
+						recordGenerator.generate(partition, Integer.toString(partition*10+key%2), ++i);
 
 				kafkaProducer.send(record, (metadata, e) ->
 				{
@@ -243,6 +260,8 @@ class ApplicationTests
 	@BeforeEach
 	public void init()
 	{
+		seekToEnd();
+
 		oldOffsets = new HashMap<>();
 		newOffsets = new HashMap<>();
 		receivedRecords = new HashSet<>();
@@ -312,5 +331,18 @@ class ApplicationTests
 
 			return new KafkaProducer<>(props);
 		}
+
+		@Bean
+		KafkaConsumer<Bytes, Bytes> offsetConsumer(ApplicationProperties properties)
+		{
+			Properties props = new Properties();
+			props.put("bootstrap.servers", properties.getBootstrapServer());
+			props.put("client.id", "OFFSET-CONSUMER");
+			props.put("group.id", properties.getGroupId());
+			props.put("key.deserializer", BytesDeserializer.class.getName());
+			props.put("value.deserializer", BytesDeserializer.class.getName());
+
+			return new KafkaConsumer<>(props);
+		}
 	}
 }