Backpressure für den `ExampleProducer` implementiert

diff --git a/README.sh b/README.sh
index 918e83abe45997051481f4c8cd8df18200dba46d..19419b57f60fb3575cf00fed14f088f6cb72221c 100755 (executable)
--- a/README.sh
+++ b/README.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-IMAGE=juplo/spring-producer:1.0-SNAPSHOT
+IMAGE=juplo/spring-producer:1.0-backpressure-SNAPSHOT
 
 if [ "$1" = "cleanup" ]
 then

diff --git a/build.gradle b/build.gradle
index 1429c4dd737dcce22957e983bcb8f67635c7a0a5..3d22e5b5db9e845da9a1f3885dbdcd62a0743b4c 100644 (file)
--- a/build.gradle
+++ b/build.gradle
@@ -8,7 +8,7 @@ plugins {
 }
 
 group = 'de.juplo.kafka'
-version = '1.0-SNAPSHOT'
+version = '1.0-backpressure-SNAPSHOT'
 
 java {
        toolchain {

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index a5fd281b6d9782daf0839280402fb32855f75c36..c3a90d670d7ab3b8ff2c03774c0d20b0b425ab47 100644 (file)
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -136,13 +136,14 @@ services:
       - kafka-3
 
   producer:
-    image: juplo/spring-producer:1.0-SNAPSHOT
+    image: juplo/spring-producer:1.0-backpressure-SNAPSHOT
     environment:
       juplo.bootstrap-server: kafka:9092
       juplo.client-id: producer
       juplo.producer.topic: test
       juplo.producer.delivery-timeout: 2147483647ms
-      juplo.producer.buffer-memory: 32768
+      juplo.producer.max-block: 2147483647ms
+      juplo.producer.max-queue-length: 10
 
   consumer:
     image: juplo/simple-consumer:1.0-SNAPSHOT

diff --git a/pom.xml b/pom.xml
index f64266b4f648b0d4c915e289da8a3e4284399f90..0360ad398d12a70a0099525b8f864e2c583da41b 100644 (file)
--- a/pom.xml
+++ b/pom.xml
@@ -15,7 +15,7 @@
   <artifactId>spring-producer</artifactId>
   <name>Spring Producer</name>
   <description>A Simple Producer, based on Spring Boot, that sends messages via Kafka</description>
-  <version>1.0-SNAPSHOT</version>
+  <version>1.0-backpressure-SNAPSHOT</version>
 
   <properties>
     <java.version>21</java.version>

diff --git a/src/main/java/de/juplo/kafka/ApplicationConfiguration.java b/src/main/java/de/juplo/kafka/ApplicationConfiguration.java
index 0090ceeab45f9b9e4a262dada2ef52a37744ba7a..5fa7140bea4c3a0a6d16b028e3e00edaebf196ff 100644 (file)
--- a/src/main/java/de/juplo/kafka/ApplicationConfiguration.java
+++ b/src/main/java/de/juplo/kafka/ApplicationConfiguration.java
@@ -29,6 +29,7 @@ public class ApplicationConfiguration
         properties.getProducerProperties().getThrottle() == null
           ? Duration.ofMillis(500)
           : properties.getProducerProperties().getThrottle(),
+        properties.getProducerProperties().getMaxQueueLength(),
         kafkaProducer,
         () -> applicationContext.close());
   }

diff --git a/src/main/java/de/juplo/kafka/ApplicationProperties.java b/src/main/java/de/juplo/kafka/ApplicationProperties.java
index 43232628755882f32117ac1a72c5ade16dd13f1e..f2fd951f37d9145d94bdb7e8f357abbc2ceb71bd 100644 (file)
--- a/src/main/java/de/juplo/kafka/ApplicationProperties.java
+++ b/src/main/java/de/juplo/kafka/ApplicationProperties.java
@@ -55,6 +55,8 @@ public class ApplicationProperties
     @NotNull
     private Duration linger;
     @NotNull
+    private Integer maxQueueLength;
+    @NotNull
     @NotEmpty
     private String compressionType;
     private Duration throttle;

diff --git a/src/main/java/de/juplo/kafka/ExampleProducer.java b/src/main/java/de/juplo/kafka/ExampleProducer.java
index c5a5a80d2d2a1798ac0c59873c5bd9ec833db5c4..006d66e88e40d0d67190c8c73be05f2a9951be93 100644 (file)
--- a/src/main/java/de/juplo/kafka/ExampleProducer.java
+++ b/src/main/java/de/juplo/kafka/ExampleProducer.java
@@ -5,6 +5,7 @@ import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 
 import java.time.Duration;
+import java.util.concurrent.atomic.AtomicInteger;
 
 
 @Slf4j
@@ -13,11 +14,13 @@ public class ExampleProducer implements Runnable
   private final String id;
   private final String topic;
   private final Duration throttle;
+  private final int maxQueueLength;
   private final Producer<String, String> producer;
   private final Thread workerThread;
   private final Runnable closeCallback;
 
   private volatile boolean running = true;
+  private final AtomicInteger queued = new AtomicInteger(0);
   private long produced = 0;
 
 
@@ -25,12 +28,14 @@ public class ExampleProducer implements Runnable
     String id,
     String topic,
     Duration throttle,
+    int maxQueueLength,
     Producer<String, String> producer,
     Runnable closeCallback)
   {
     this.id = id;
     this.topic = topic;
     this.throttle = throttle;
+    this.maxQueueLength = maxQueueLength;
     this.producer = producer;
 
     workerThread = new Thread(this, "ExampleProducer Worker-Thread");
@@ -80,6 +85,18 @@ public class ExampleProducer implements Runnable
 
   void send(String key, String value)
   {
+    while (queued.get() >= maxQueueLength)
+    {
+      try
+      {
+        Thread.sleep(1000);
+      }
+      catch (InterruptedException e)
+      {
+        log.warn("{} - Interrupted while waiting for queue to be progressed, queued={}!", queued, e);
+      }
+    }
+
     final long time = System.currentTimeMillis();
 
     final ProducerRecord<String, String> record = new ProducerRecord<>(
@@ -88,32 +105,37 @@ public class ExampleProducer implements Runnable
       value   // Value
     );
 
+    int queuedAfterSend = queued.incrementAndGet();
+
     producer.send(record, (metadata, e) ->
     {
       long now = System.currentTimeMillis();
+      int queuedAfterReceive = queued.decrementAndGet();
       if (e == null)
       {
         // HANDLE SUCCESS
         log.debug(
-          "{} - Sent message {}={}, partition={}, offset={}, timestamp={}, latency={}ms",
+          "{} - Sent message {}={}, partition={}, offset={}, timestamp={}, latency={}ms, queued={}",
           id,
           key,
           value,
           metadata.partition(),
           metadata.offset(),
           metadata.timestamp(),
-          now - time
+          now - time,
+          queuedAfterReceive
         );
       }
       else
       {
         // HANDLE ERROR
         log.error(
-          "{} - ERROR for message {}={}, latency={}ms: {}",
+          "{} - ERROR for message {}={}, latency={}ms, queued={}: {}",
           id,
           key,
           value,
           now - time,
+          queuedAfterReceive,
           e.toString()
         );
       }
@@ -122,11 +144,12 @@ public class ExampleProducer implements Runnable
     long now = System.currentTimeMillis();
     produced++;
     log.trace(
-      "{} - Queued message {}={}, latency={}ms",
+      "{} - Queued message {}={}, latency={}ms, queued={}",
       id,
       key,
       value,
-      now - time
+      now - time,
+      queuedAfterSend
     );
   }
 

diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 21fef284f03e091e3a9e7f87758e5fef9fb2fe42..5d519757d7356c8e55b7cb7504ba3554eed98064 100644 (file)
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -9,6 +9,7 @@ juplo:
     buffer-memory: 33554432
     batch-size: 16384
     linger: 0
+    max-queue-length: 100
     compression-type: gzip
     throttle: 500ms
 management:
@@ -36,6 +37,7 @@ info:
       buffer-memory: ${juplo.producer.buffer-memory}
       batch-size: ${juplo.producer.batch-size}
       linger: ${juplo.producer.linger}
+      max-queue-length: ${juplo.producer.max-queue-length}
       compression-type: ${juplo.producer.compression-type}
       throttle: ${juplo.producer.throttle}
 logging: