From: Kai Moritz <kai@juplo.de>
Date: Tue, 14 May 2024 20:44:21 +0000 (+0200)
Subject: WIP
X-Git-Url: https://juplo.de/gitweb/?p=demos%2Fkafka%2Fwordcount;a=commitdiff_plain;h=HEAD;hp=827cd286933a1af205ee9fb9224f94373518158e;ds=sidebyside

WIP
---

diff --git a/.gitmodules b/.gitmodules
index ae65594..b8abc8d 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -26,3 +26,6 @@
 	path = fortune
 	url = ./
 	branch = fortune
+[submodule "splitter"]
+	path = splitter
+	url = ./
diff --git a/README.sh b/README.sh
index 4b5f557..c7e8086 100755
--- a/README.sh
+++ b/README.sh
@@ -1,18 +1,20 @@
 #!/bin/bash
 
 FORTUNE=juplo/wordcount--fortune:1.0.0
-RECORDER=juplo/wordcount--recorder:1.0.1
-USERS=juplo/wordcount--users:1.0.4
-COUNTER=juplo/wordcount--counter:1.0.1
-TOP10=juplo/wordcount--top10:1.0.1
-QUERY=juplo/wordcount--query:1.0.4
+RECORDER=juplo/wordcount--recorder:1.1.1
+USERS=juplo/wordcount--users:1.0.5
+SPLITTER=juplo/wordcount--splitter:1.1.5
+COUNTER=juplo/wordcount--counter:1.2.14
+TOP10=juplo/wordcount--top10:1.0.2
+QUERY=juplo/wordcount--query:1.0.5
 
 if [ "$1" = "cleanup" ]
 then
-  docker-compose down -v --remove-orphans
+  docker compose down -v --remove-orphans
   docker image rm $FORTUNE
   docker image rm $RECORDER
   docker image rm $USERS
+  docker image rm $SPLITTER
   docker image rm $COUNTER
   docker image rm $TOP10
   docker image rm $QUERY
@@ -23,6 +25,7 @@ if [[
   $(docker image ls -q $FORTUNE) == "" ||
   $(docker image ls -q $RECORDER) == "" ||
   $(docker image ls -q $USERS) == "" ||
+  $(docker image ls -q $SPLITTER) == "" ||
   $(docker image ls -q $COUNTER) == "" ||
   $(docker image ls -q $TOP10) == "" ||
   $(docker image ls -q $QUERY) == "" ||
@@ -30,10 +33,11 @@ if [[
 ]]
 then
   mvn clean package || exit
-  docker-compose rm -svf recorder users counter top10 query
+  docker compose rm -svf recorder users splitter counter top10 query
   ( cd fortune; ./README.sh $FORTUNE; )
   mvn -f recorder/pom.xml docker:build
   mvn -f users/pom.xml docker:build
+  mvn -f splitter/pom.xml docker:build
   mvn -f counter/pom.xml docker:build
   mvn -f top10/pom.xml docker:build
   mvn -f query/pom.xml docker:build
@@ -42,26 +46,26 @@ else
   docker image ls $FORTUNE
   docker image ls $RECORDER
   docker image ls $USERS
+  docker image ls $SPLITTER
   docker image ls $COUNTER
   docker image ls $TOP10
   docker image ls $QUERY
 fi
 
 
-docker-compose up -d zookeeper kafka cli traefik
+docker compose up -d kafka cli traefik
 
 echo "Waiting for the Kafka-Cluster to become ready..."
-docker-compose exec cli cub kafka-ready -b kafka:9092 1 60 > /dev/null 2>&1 || exit 1
-docker-compose exec cli zookeeper-shell zookeeper:2181 ls /brokers/ids
+docker compose exec cli cub kafka-ready -b kafka:9092 1 60 > /dev/null 2>&1 || exit 1
 
-docker-compose up -d users
+docker compose up -d users
 while [ "$(http users.localhost/actuator/health 2> /dev/null | jq -r .status 2> /dev/null)" != "UP" ]; do echo "Waiting for service users..."; sleep 1; done
 http -v post users.localhost username=bart firstName=Bart lastName=Simpson sex=MALE
 http -v post users.localhost username=nerd firstName=Jane lastName=Doe sex=FEMALE
 http -v post users.localhost username=riddler firstName=Rumpel lastName=Stilzchen sex=MALE
 
 echo "Starting all instances..."
-docker-compose up -d
+docker compose up -d
 
 while [ "$(http query.localhost/actuator/health 2>/dev/null | jq -r .status 2> /dev/null)" != "UP" ]; do echo "Waiting for service query..."; sleep 1; done
 http -v query.localhost/bart
@@ -80,7 +84,7 @@ http -v query.localhost/bart
 http -v query.localhost/nerd
 http -v query.localhost/riddler
 
-docker-compose up -d --scale query=5 query
+# docker compose up -d --scale query=5 query
 http -v --follow query.localhost/bart
 sleep 1
 http -v --follow query.localhost/bart
diff --git a/counter b/counter
index b1623c2..1b1ac5b 160000
--- a/counter
+++ b/counter
@@ -1 +1 @@
-Subproject commit b1623c2b645c614424ceec3bc71008c900d82115
+Subproject commit 1b1ac5bca62f07dae4188a5797a36b0fbe72058d
diff --git a/docker-compose.yml b/docker-compose.yml
index 87338db..0252da4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,29 +1,26 @@
-version: '3.2'
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:6.2.0
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-    ports:
-      - 2181:2181
 
   kafka:
-    image: confluentinc/cp-kafka:6.2.0
+    image: confluentinc/cp-kafka:7.5.1
     environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENERS: DOCKER://:9092, LOCALHOST://:9082
+      KAFKA_NODE_ID: 1
+      KAFKA_LISTENERS: CONTROLLER://:9072, DOCKER://:9092, LOCALHOST://:9082
       KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka:9092, LOCALHOST://localhost:9082
       KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT, DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_PROCESS_ROLES: broker, controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9072
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
     ports:
       - 9092:9082
       - 9082:9082
-    depends_on:
-      - zookeeper
-
+    stop_grace_period: 120s
 
   traefik:
     image: "traefik:v2.5"
@@ -40,7 +37,7 @@ services:
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
 
   recorder:
-    image: juplo/wordcount--recorder:1.0.1
+    image: juplo/wordcount--recorder:1.1.1
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.recorder.rule=Host(`recorder.localhost`)"
@@ -51,7 +48,7 @@ services:
       - kafka
 
   users:
-    image: juplo/wordcount--users:1.0.4
+    image: juplo/wordcount--users:1.0.5
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.users.rule=Host(`users.localhost`)"
@@ -61,8 +58,19 @@ services:
     depends_on:
       - kafka
 
+  splitter:
+    image: juplo/wordcount--splitter:1.1.5
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.splitter.rule=Host(`splitter.localhost`)"
+      - "traefik.http.routers.splitter.entrypoints=web"
+    environment:
+      juplo.wordcount.splitter.bootstrap-server: kafka:9092
+    depends_on:
+      - kafka
+
   counter:
-    image: juplo/wordcount--counter:1.0.1
+    image: juplo/wordcount--counter:1.2.14
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.counter.rule=Host(`counter.localhost`)"
@@ -73,7 +81,7 @@ services:
       - kafka
 
   top10:
-    image: juplo/wordcount--top10:1.0.1
+    image: juplo/wordcount--top10:1.0.2
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.top10.rule=Host(`top10.localhost`)"
@@ -84,7 +92,7 @@ services:
       - kafka
 
   query:
-    image: juplo/wordcount--query:1.0.4
+    image: juplo/wordcount--query:1.0.5
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.query.rule=Host(`query.localhost`)"
@@ -134,10 +142,11 @@ services:
     image: juplo/toolbox
     command: bash -c "
       cub kafka-ready -b kafka:9092 1 60 ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --topic recordings ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --topic users ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --topic countings ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --topic top10 ;
+      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic recordings ;
+      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic users ;
+      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic words ;
+      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic countings ;
+      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic top10 ;
       sleep infinity"
 
   akhq:
@@ -155,8 +164,3 @@ services:
             docker-kafka-server:
               properties:
                 bootstrap.servers: "kafka:9092"
-
-networks:
-  default:
-    external:
-      name: trion
diff --git a/fortune b/fortune
index 5dd5c42..dde18ef 160000
--- a/fortune
+++ b/fortune
@@ -1 +1 @@
-Subproject commit 5dd5c422829ceac252d5c13d0022acff40f76185
+Subproject commit dde18ef24d54bb988ad458ad6e12ed99fbc53061
diff --git a/pom.xml b/pom.xml
index 606ca8b..e116624 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,13 +5,14 @@
 	<groupId>de.juplo.kafka.wordcount</groupId>
 	<artifactId>docker</artifactId>
 	<packaging>pom</packaging>
-	<version>1.0.1</version>
+	<version>1.0.2</version>
 	<name>Docker-Compose Setup</name>
 	<description>Docker-Compose Setup for the multiuser-wordcount example</description>
 
 	<modules>
 		<module>recorder</module>
 		<module>users</module>
+		<module>splitter</module>
 		<module>counter</module>
 		<module>top10</module>
 		<module>query</module>
diff --git a/query b/query
index 251cf79..3e27e5d 160000
--- a/query
+++ b/query
@@ -1 +1 @@
-Subproject commit 251cf79927d4c1c941467c8fee0349c1a7eb4b93
+Subproject commit 3e27e5d91b4920ffd3cd061595329b5499c3389f
diff --git a/recorder b/recorder
index c5700c2..7a5943c 160000
--- a/recorder
+++ b/recorder
@@ -1 +1 @@
-Subproject commit c5700c2117f6c445278f272572c8b5732bf53bbf
+Subproject commit 7a5943ce46684c9b469f38780d95955a9795044c
diff --git a/splitter b/splitter
new file mode 160000
index 0000000..36bd7f7
--- /dev/null
+++ b/splitter
@@ -0,0 +1 @@
+Subproject commit 36bd7f7a03be33a05f371ebb9dc7b398678a08dc
diff --git a/top10 b/top10
index b3c3b1c..1bfbd6c 160000
--- a/top10
+++ b/top10
@@ -1 +1 @@
-Subproject commit b3c3b1c1ab19d197d323d7c69e03d9bd09147690
+Subproject commit 1bfbd6c6fe3c35e0cd5470db632faf1a655487af
diff --git a/users b/users
index 097a537..aa0d1c4 160000
--- a/users
+++ b/users
@@ -1 +1 @@
-Subproject commit 097a537e42272d105354a708e5138abf5488b28a
+Subproject commit aa0d1c4c8e47b2fffc5fa0ca2cae83f64ed8dd4e