X-Git-Url: https://juplo.de/gitweb/?a=blobdiff_plain;ds=sidebyside;f=src%2Ftest%2Fjava%2Fde%2Fjuplo%2Fkafka%2FApplicationTests.java;h=09614b8e23702f729214dd33bc7820ad3c83208d;hb=c808810e9e33afe33b29f7fd3921023ecd15483d;hp=aa6dd4d21afd5f17ebf3aaf5640062318f4f2c44;hpb=a6a0a22a5fa34a01b0e8b2bc1e0e2b82d7b60f33;p=demos%2Fkafka%2Ftraining

diff --git a/src/test/java/de/juplo/kafka/ApplicationTests.java b/src/test/java/de/juplo/kafka/ApplicationTests.java
index aa6dd4d..09614b8 100644
--- a/src/test/java/de/juplo/kafka/ApplicationTests.java
+++ b/src/test/java/de/juplo/kafka/ApplicationTests.java
@@ -20,14 +20,11 @@ import org.springframework.kafka.test.context.EmbeddedKafka;
 import org.springframework.test.context.TestPropertySource;
 import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
 
-import java.time.Clock;
 import java.time.Duration;
 import java.util.*;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.function.BiConsumer;
-import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
@@ -64,6 +61,8 @@ class ApplicationTests
 	@Autowired
 	KafkaConsumer<String, String> kafkaConsumer;
 	@Autowired
+	KafkaConsumer<Bytes, Bytes> offsetConsumer;
+	@Autowired
 	PartitionStatisticsRepository partitionStatisticsRepository;
 	@Autowired
 	ApplicationProperties properties;
@@ -71,9 +70,12 @@ class ApplicationTests
 	ExecutorService executor;
 	@Autowired
 	PartitionStatisticsRepository repository;
+	@Autowired
+	SumRebalanceListener sumRebalanceListener;
+	@Autowired
+	SumRecordHandler sumRecordHandler;
 
-	Consumer<ConsumerRecord<String, String>> testHandler;
-	EndlessConsumer endlessConsumer;
+	EndlessConsumer<String, String> endlessConsumer;
 	Map<TopicPartition, Long> oldOffsets;
 	Map<TopicPartition, Long> newOffsets;
 	Set<ConsumerRecord<String, String>> receivedRecords;
@@ -84,14 +86,20 @@ class ApplicationTests
 	@Test
 	void commitsCurrentOffsetsOnSuccess() throws ExecutionException, InterruptedException
 	{
-		send100Messages(i ->  new Bytes(valueSerializer.serialize(TOPIC, i)));
+		send100Messages((partition, key, counter) ->
+		{
+			Bytes value = new Bytes(valueSerializer.serialize(TOPIC, counter));
+			return new ProducerRecord<>(TOPIC, partition, key, value);
+		});
 
 		await("100 records received")
 				.atMost(Duration.ofSeconds(30))
+				.pollInterval(Duration.ofSeconds(1))
 				.until(() -> receivedRecords.size() >= 100);
 
 		await("Offsets committed")
 				.atMost(Duration.ofSeconds(10))
+				.pollInterval(Duration.ofSeconds(1))
 				.untilAsserted(() ->
 				{
 					checkSeenOffsetsForProgress();
@@ -124,8 +132,8 @@ class ApplicationTests
 		Set<TopicPartition> withProgress = new HashSet<>();
 		partitions().forEach(tp ->
 		{
-			Long oldOffset = oldOffsets.get(tp);
-			Long newOffset = newOffsets.get(tp);
+			Long oldOffset = oldOffsets.get(tp) + 1;
+			Long newOffset = newOffsets.get(tp) + 1;
 			if (!oldOffset.equals(newOffset))
 			{
 				log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
@@ -140,6 +148,24 @@ class ApplicationTests
 
 	/** Helper methods for setting up and running the tests */
 
+	void seekToEnd()
+	{
+		offsetConsumer.assign(partitions());
+		partitions().forEach(tp ->
+		{
+			Long offset = offsetConsumer.position(tp);
+			log.info("New position for {}: {}", tp, offset);
+			Integer partition = tp.partition();
+			StatisticsDocument document =
+					partitionStatisticsRepository
+							.findById(partition.toString())
+							.orElse(new StatisticsDocument(partition));
+			document.offset = offset;
+			partitionStatisticsRepository.save(document);
+		});
+		offsetConsumer.unsubscribe();
+	}
+
 	void doForCurrentOffsets(BiConsumer<TopicPartition, Long> consumer)
 	{
 		partitions().forEach(tp ->
@@ -160,7 +186,12 @@ class ApplicationTests
 	}
 
 
-	void send100Messages(Function<Long, Bytes> messageGenerator)
+	public interface RecordGenerator<K, V>
+	{
+		public ProducerRecord<String, Bytes> generate(int partition, String key, long counter);
+	}
+
+	void send100Messages(RecordGenerator recordGenerator)
 	{
 		long i = 0;
 
@@ -168,14 +199,8 @@ class ApplicationTests
 		{
 			for (int key = 0; key < 10; key++)
 			{
-				Bytes value = messageGenerator.apply(++i);
-
 				ProducerRecord<String, Bytes> record =
-						new ProducerRecord<>(
-								TOPIC,
-								partition,
-								Integer.toString(key%2),
-								value);
+						recordGenerator.generate(partition, Integer.toString(partition*10+key%2), ++i);
 
 				kafkaProducer.send(record, (metadata, e) ->
 				{
@@ -205,7 +230,7 @@ class ApplicationTests
 	@BeforeEach
 	public void init()
 	{
-		testHandler = record -> {} ;
+		seekToEnd();
 
 		oldOffsets = new HashMap<>();
 		newOffsets = new HashMap<>();
@@ -217,25 +242,26 @@ class ApplicationTests
 			newOffsets.put(tp, offset - 1);
 		});
 
-		Consumer<ConsumerRecord<String, String>> captureOffsetAndExecuteTestHandler =
-				record ->
-				{
-					newOffsets.put(
-							new TopicPartition(record.topic(), record.partition()),
-							record.offset());
-					receivedRecords.add(record);
-					testHandler.accept(record);
+		TestRecordHandler<String, String> captureOffsetAndExecuteTestHandler =
+				new TestRecordHandler<String, String>(sumRecordHandler) {
+					@Override
+					public void onNewRecord(ConsumerRecord<String, String> record)
+					{
+						newOffsets.put(
+								new TopicPartition(record.topic(), record.partition()),
+								record.offset());
+						receivedRecords.add(record);
+					}
 				};
 
 		endlessConsumer =
-				new EndlessConsumer(
+				new EndlessConsumer<>(
 						executor,
-						repository,
 						properties.getClientId(),
 						properties.getTopic(),
-						Clock.systemDefaultZone(),
-						properties.getCommitInterval(),
-						kafkaConsumer);
+						kafkaConsumer,
+						sumRebalanceListener,
+						captureOffsetAndExecuteTestHandler);
 
 		endlessConsumer.start();
 	}
@@ -275,5 +301,19 @@ class ApplicationTests
 
 			return new KafkaProducer<>(props);
 		}
+
+		@Bean
+		KafkaConsumer<Bytes, Bytes> offsetConsumer(ApplicationProperties properties)
+		{
+			Properties props = new Properties();
+			props.put("bootstrap.servers", properties.getBootstrapServer());
+			props.put("client.id", "OFFSET-CONSUMER");
+			props.put("enable.auto.commit", false);
+			props.put("auto.offset.reset", "latest");
+			props.put("key.deserializer", BytesDeserializer.class.getName());
+			props.put("value.deserializer", BytesDeserializer.class.getName());
+
+			return new KafkaConsumer<>(props);
+		}
 	}
 }