From: Kai Moritz <kai@juplo.de>
Date: Tue, 25 Jun 2024 05:09:10 +0000 (+0200)
Subject: WIP
X-Git-Url: https://juplo.de/gitweb/?a=commitdiff_plain;ds=inline;h=HEAD;hp=1d8af593dfe7ec0e067916bcf08107352d708561;p=demos%2Fkafka%2Fwordcount

WIP
---

diff --git a/.gitmodules b/.gitmodules
index b8abc8d..aa65726 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -29,3 +29,11 @@
 [submodule "splitter"]
 	path = splitter
 	url = ./
+[submodule "popular"]
+	path = popular
+	url = ./
+	branch = popular
+[submodule "stats"]
+	path = stats
+	url = ./
+	branch = stats
diff --git a/README.sh b/README.sh
index ba50a69..62805c6 100755
--- a/README.sh
+++ b/README.sh
@@ -1,16 +1,17 @@
 #!/bin/bash
 
-FORTUNE=juplo/wordcount--fortune:1.0.0
-RECORDER=juplo/wordcount--recorder:1.0.1
-USERS=juplo/wordcount--users:1.0.4
-SPLITTER=juplo/wordcount--splitter:1.0.0
-COUNTER=juplo/wordcount--counter:1.1.0
-TOP10=juplo/wordcount--top10:1.0.1
-QUERY=juplo/wordcount--query:1.0.4
+FORTUNE=juplo/wordcount--fortune:1.0.1
+RECORDER=juplo/wordcount--recorder:1.2.2
+USERS=juplo/wordcount--users:1.0.7
+SPLITTER=juplo/wordcount--splitter:1.2.2
+COUNTER=juplo/wordcount--counter:1.4.2
+TOP10=juplo/wordcount--top10:1.4.2
+QUERY=juplo/wordcount--query:2.1.2
+POPULAR=juplo/wordcount--popular:1.3.2
 
 if [ "$1" = "cleanup" ]
 then
-  docker-compose down -v --remove-orphans
+  docker compose down -v --remove-orphans
   docker image rm $FORTUNE
   docker image rm $RECORDER
   docker image rm $USERS
@@ -18,6 +19,7 @@ then
   docker image rm $COUNTER
   docker image rm $TOP10
   docker image rm $QUERY
+  docker image rm $POPULAR
   exit
 fi
 
@@ -29,44 +31,104 @@ if [[
   $(docker image ls -q $COUNTER) == "" ||
   $(docker image ls -q $TOP10) == "" ||
   $(docker image ls -q $QUERY) == "" ||
+  $(docker image ls -q $POPULAR) == "" ||
   "$1" = "build"
 ]]
 then
-  mvn clean package || exit
-  docker-compose rm -svf recorder users splitter counter top10 query
+  docker compose rm -svf bart riddler nerd recorder users splitter counter top10 query popular
+fi
+
+if [[ $(docker image ls -q $FORTUNE) == "" || "$1" = "build" ]]
+then
+  echo "Building image $FORTUNE"
   ( cd fortune; ./README.sh $FORTUNE; )
-  mvn -f recorder/pom.xml docker:build
-  mvn -f users/pom.xml docker:build
-  mvn -f splitter/pom.xml docker:build
-  mvn -f counter/pom.xml docker:build
-  mvn -f top10/pom.xml docker:build
-  mvn -f query/pom.xml docker:build
 else
-  echo "Using existing images:"
-  docker image ls $FORTUNE
-  docker image ls $RECORDER
-  docker image ls $USERS
-  docker image ls $SPLITTER
-  docker image ls $COUNTER
-  docker image ls $TOP10
-  docker image ls $QUERY
+  echo -n "Using existing image for $FORTUNE: "
+  docker image ls --format json $FORTUNE | jq -r '(.ID + " - " + .CreatedSince)'
+fi
+
+if [[ $(docker image ls -q $RECORDER) == "" || "$1" = "build" ]]
+then
+  echo "Building image $RECORDER"
+  mvn -f recorder/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $RECORDER: "
+  docker image ls --format json $RECORDER | jq -r '(.ID + " - " + .CreatedSince)'
 fi
 
+if [[ $(docker image ls -q $USERS) == "" || "$1" = "build" ]]
+then
+  echo "Building image $USERS"
+  mvn -f users/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $USERS: "
+  docker image ls --format json $USERS | jq -r '(.ID + " - " + .CreatedSince)'
+fi
 
-docker-compose up -d zookeeper kafka cli traefik
+if [[ $(docker image ls -q $SPLITTER) == "" || "$1" = "build" ]]
+then
+  echo "Building image $SPLITTER"
+  mvn -f splitter/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $SPLITTER: "
+  docker image ls --format json $SPLITTER | jq -r '(.ID + " - " + .CreatedSince)'
+fi
 
-echo "Waiting for the Kafka-Cluster to become ready..."
-docker-compose exec cli cub kafka-ready -b kafka:9092 1 60 > /dev/null 2>&1 || exit 1
-docker-compose exec cli zookeeper-shell zookeeper:2181 ls /brokers/ids
+if [[ $(docker image ls -q $COUNTER) == "" || "$1" = "build" ]]
+then
+  echo "Building image $COUNTER"
+  mvn -f counter/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $COUNTER: "
+  docker image ls --format json $COUNTER | jq -r '(.ID + " - " + .CreatedSince)'
+fi
 
-docker-compose up -d users
+if [[ $(docker image ls -q $TOP10) == "" || "$1" = "build" ]]
+then
+  echo "Building image $TOP10"
+  mvn -f top10/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $TOP10: "
+  docker image ls --format json $TOP10 | jq -r '(.ID + " - " + .CreatedSince)'
+fi
+
+if [[ $(docker image ls -q $QUERY) == "" || "$1" = "build" ]]
+then
+  echo "Building image $QUERY"
+  mvn -f query/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $QUERY: "
+  docker image ls --format json $QUERY | jq -r '(.ID + " - " + .CreatedSince)'
+fi
+
+if [[ $(docker image ls -q $POPULAR) == "" || "$1" = "build" ]]
+then
+  echo "Building image $POPULAR"
+  mvn -f popular/pom.xml clean package docker:build
+else
+  echo -n "Using existing image for $POPULAR: "
+  docker image ls --format json $POPULAR | jq -r '(.ID + " - " + .CreatedSince)'
+fi
+
+
+docker compose up setup
+
+echo "Starting the service 'users'..."
+docker compose up -d users
 while [ "$(http users.localhost/actuator/health 2> /dev/null | jq -r .status 2> /dev/null)" != "UP" ]; do echo "Waiting for service users..."; sleep 1; done
 http -v post users.localhost username=bart firstName=Bart lastName=Simpson sex=MALE
 http -v post users.localhost username=nerd firstName=Jane lastName=Doe sex=FEMALE
 http -v post users.localhost username=riddler firstName=Rumpel lastName=Stilzchen sex=MALE
 
-echo "Starting all instances..."
-docker-compose up -d
+echo "Starting the other instances..."
+docker compose up -d recorder
+docker compose up -d splitter
+docker compose up -d counter
+docker compose up -d top10
+docker compose up -d query
+
+echo "Starting the simulated users..."
+docker compose up -d bart nerd riddler
 
 while [ "$(http query.localhost/actuator/health 2>/dev/null | jq -r .status 2> /dev/null)" != "UP" ]; do echo "Waiting for service query..."; sleep 1; done
 http -v query.localhost/bart
@@ -85,7 +147,7 @@ http -v query.localhost/bart
 http -v query.localhost/nerd
 http -v query.localhost/riddler
 
-// docker-compose up -d --scale query=5 query
+# docker compose up -d --scale query=5 query
 http -v --follow query.localhost/bart
 sleep 1
 http -v --follow query.localhost/bart
diff --git a/counter b/counter
index fe04994..996232b 160000
--- a/counter
+++ b/counter
@@ -1 +1 @@
-Subproject commit fe049944648d9675ffe66b586b6bf860fbac696f
+Subproject commit 996232bdbc8deeca105854f3f04d66715ecfa269
diff --git a/docker-compose.yml b/docker-compose.yml
index c02465a..c8ec5a2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,29 +1,26 @@
-version: '3.2'
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:6.2.0
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-    ports:
-      - 2181:2181
 
   kafka:
-    image: confluentinc/cp-kafka:6.2.0
+    image: confluentinc/cp-kafka:7.5.1
     environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_LISTENERS: DOCKER://:9092, LOCALHOST://:9082
+      KAFKA_NODE_ID: 1
+      KAFKA_LISTENERS: CONTROLLER://:9072, DOCKER://:9092, LOCALHOST://:9082
       KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka:9092, LOCALHOST://localhost:9082
       KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT, DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT
+      KAFKA_PROCESS_ROLES: broker, controller
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9072
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
     ports:
       - 9092:9082
       - 9082:9082
-    depends_on:
-      - zookeeper
-
+    stop_grace_period: 120s
 
   traefik:
     image: "traefik:v2.5"
@@ -39,8 +36,9 @@ services:
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock:ro"
 
+
   recorder:
-    image: juplo/wordcount--recorder:1.0.1
+    image: juplo/wordcount--recorder:1.2.2
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.recorder.rule=Host(`recorder.localhost`)"
@@ -51,7 +49,7 @@ services:
       - kafka
 
   users:
-    image: juplo/wordcount--users:1.0.4
+    image: juplo/wordcount--users:1.0.7
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.users.rule=Host(`users.localhost`)"
@@ -62,7 +60,7 @@ services:
       - kafka
 
   splitter:
-    image: juplo/wordcount--splitter:1.0.0
+    image: juplo/wordcount--splitter:1.2.2
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.splitter.rule=Host(`splitter.localhost`)"
@@ -73,7 +71,7 @@ services:
       - kafka
 
   counter:
-    image: juplo/wordcount--counter:1.1.0
+    image: juplo/wordcount--counter:1.4.2
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.counter.rule=Host(`counter.localhost`)"
@@ -84,7 +82,7 @@ services:
       - kafka
 
   top10:
-    image: juplo/wordcount--top10:1.0.1
+    image: juplo/wordcount--top10:1.4.2
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.top10.rule=Host(`top10.localhost`)"
@@ -95,7 +93,7 @@ services:
       - kafka
 
   query:
-    image: juplo/wordcount--query:1.0.4
+    image: juplo/wordcount--query:2.1.2
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.query.rule=Host(`query.localhost`)"
@@ -105,8 +103,20 @@ services:
     depends_on:
       - kafka
 
+  popular:
+    image: juplo/wordcount--popular:1.3.2
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.popular.rule=Host(`popular.localhost`)"
+      - "traefik.http.routers.popular.entrypoints=web"
+    environment:
+      juplo.wordcount.popular.bootstrap-server: kafka:9092
+    depends_on:
+      - kafka
+
+
   bart:
-    image: juplo/wordcount--fortune:1.0.0
+    image: juplo/wordcount--fortune:1.0.1
     command: bash -c "
       while [ true ];
       do
@@ -118,7 +128,7 @@ services:
       done"
 
   nerd:
-    image: juplo/wordcount--fortune:1.0.0
+    image: juplo/wordcount--fortune:1.0.1
     command: bash -c "
       while [ true ];
       do
@@ -130,7 +140,7 @@ services:
       done"
 
   riddler:
-    image: juplo/wordcount--fortune:1.0.0
+    image: juplo/wordcount--fortune:1.0.1
     command: bash -c "
       while [ true ];
       do
@@ -141,16 +151,110 @@ services:
         sleep 1;
       done"
 
+
   cli:
     image: juplo/toolbox
-    command: bash -c "
-      cub kafka-ready -b kafka:9092 1 60 ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic recordings ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic users ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic words ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic countings ;
-      kafka-topics --bootstrap-server kafka:9092 --create --partitions 10 --if-not-exists --topic top10 ;
-      sleep infinity"
+    command: sleep infinity
+    stop_grace_period: 0s
+    depends_on:
+      - kafka
+      - traefik
+
+  setup:
+    image: juplo/toolbox
+    command:
+      - bash
+      - -c
+      - |
+        cub kafka-ready -b kafka:9092 1 60 > /dev/null 2>&1 || exit 1
+        if [ -e INITIALIZED ]
+        then
+          echo -n Bereits konfiguriert:
+          cat INITIALIZED
+          kafka-topics --bootstrap-server kafka:9092 --describe --topic users
+          kafka-topics --bootstrap-server kafka:9092 --describe --topic recordings
+          kafka-topics --bootstrap-server kafka:9092 --describe --topic words
+          kafka-topics --bootstrap-server kafka:9092 --describe --topic countings
+          kafka-topics --bootstrap-server kafka:9092 --describe --topic top10
+          kafka-topics --bootstrap-server kafka:9092 --describe --topic popular
+        else
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --delete \
+                       --if-exists \
+                       --topic users
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --delete \
+                       --if-exists \
+                       --topic recordings
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --delete \
+                       --if-exists \
+                       --topic words
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --delete \
+                       --if-exists \
+                       --topic countings
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --delete \
+                       --if-exists \
+                       --topic top10
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --delete \
+                       --if-exists \
+                       --topic popular
+          kafka-topics --bootstrap-server kafka:9092 \
+                       --create \
+                       --topic users \
+                       --partitions 3 \
+                       --replication-factor 1 \
+                       --config min.insync.replicas=1 \
+          && echo Das Topic \'users\' wurde erfolgreich angelegt: \
+          && kafka-topics --bootstrap-server kafka:9092 --describe --topic users \
+          && kafka-topics --bootstrap-server kafka:9092 \
+                       --create \
+                       --topic recordings \
+                       --partitions 3 \
+                       --replication-factor 1 \
+                       --config min.insync.replicas=1 \
+          && echo Das Topic \'recordings\' wurde erfolgreich angelegt: \
+          && kafka-topics --bootstrap-server kafka:9092 --describe --topic recordings \
+          && kafka-topics --bootstrap-server kafka:9092 \
+                       --create \
+                       --topic words \
+                       --partitions 3 \
+                       --replication-factor 1 \
+                       --config min.insync.replicas=1 \
+          && echo Das Topic \'words\' wurde erfolgreich angelegt: \
+          && kafka-topics --bootstrap-server kafka:9092 --describe --topic words \
+          && kafka-topics --bootstrap-server kafka:9092 \
+                       --create \
+                       --topic countings \
+                       --partitions 3 \
+                       --replication-factor 1 \
+                       --config min.insync.replicas=1 \
+          && echo Das Topic \'countings\' wurde erfolgreich angelegt: \
+          && kafka-topics --bootstrap-server kafka:9092 --describe --topic countings \
+          && kafka-topics --bootstrap-server kafka:9092 \
+                       --create \
+                       --topic top10 \
+                       --partitions 3 \
+                       --replication-factor 1 \
+                       --config min.insync.replicas=1 \
+          && echo Das Topic \'top10\' wurde erfolgreich angelegt: \
+          && kafka-topics --bootstrap-server kafka:9092 --describe --topic top10 \
+          && kafka-topics --bootstrap-server kafka:9092 \
+                       --create \
+                       --topic popular \
+                       --partitions 3 \
+                       --replication-factor 1 \
+                       --config min.insync.replicas=1 \
+          && echo Das Topic \'popular\' wurde erfolgreich angelegt: \
+          && kafka-topics --bootstrap-server kafka:9092 --describe --topic popular \
+          && date > INITIALIZED
+        fi
+    stop_grace_period: 0s
+    depends_on:
+      - cli
 
   akhq:
     image: tchiotludo/akhq:0.18.0
diff --git a/fortune b/fortune
index dde18ef..8c69a7c 160000
--- a/fortune
+++ b/fortune
@@ -1 +1 @@
-Subproject commit dde18ef24d54bb988ad458ad6e12ed99fbc53061
+Subproject commit 8c69a7caad51f067f129a751966d0f4638fa228c
diff --git a/pom.xml b/pom.xml
index e116624..fe057b1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
 	<groupId>de.juplo.kafka.wordcount</groupId>
 	<artifactId>docker</artifactId>
 	<packaging>pom</packaging>
-	<version>1.0.2</version>
+	<version>2.0.1</version>
 	<name>Docker-Compose Setup</name>
 	<description>Docker-Compose Setup for the multiuser-wordcount example</description>
 
@@ -16,5 +16,6 @@
 		<module>counter</module>
 		<module>top10</module>
 		<module>query</module>
+		<module>popular</module>
 	</modules>
 </project>
diff --git a/popular b/popular
new file mode 160000
index 0000000..b5adc17
--- /dev/null
+++ b/popular
@@ -0,0 +1 @@
+Subproject commit b5adc17bf44e22e5cdfabf7e82ab433de44d1abd
diff --git a/query b/query
index 251cf79..9c1b637 160000
--- a/query
+++ b/query
@@ -1 +1 @@
-Subproject commit 251cf79927d4c1c941467c8fee0349c1a7eb4b93
+Subproject commit 9c1b63783704ee2adf031441ad13bfaba65f5969
diff --git a/recorder b/recorder
index c5700c2..3a77a50 160000
--- a/recorder
+++ b/recorder
@@ -1 +1 @@
-Subproject commit c5700c2117f6c445278f272572c8b5732bf53bbf
+Subproject commit 3a77a501fb93403b6048df4e89c0ba515ac02242
diff --git a/splitter b/splitter
index 3bc3ef7..fa9b1d3 160000
--- a/splitter
+++ b/splitter
@@ -1 +1 @@
-Subproject commit 3bc3ef7ceb76e7643e1e373293cfdc78f870838a
+Subproject commit fa9b1d34466c3cc8f8daaaefb926f19a1b7f5251
diff --git a/stats b/stats
new file mode 160000
index 0000000..28f7fff
--- /dev/null
+++ b/stats
@@ -0,0 +1 @@
+Subproject commit 28f7fff6226d6f01f2c06b9d43c4b62aaf2c9ebe
diff --git a/top10 b/top10
index b3c3b1c..e4f7f1e 160000
--- a/top10
+++ b/top10
@@ -1 +1 @@
-Subproject commit b3c3b1c1ab19d197d323d7c69e03d9bd09147690
+Subproject commit e4f7f1e134ae40ac8d4002a742f8426327cf177b
diff --git a/users b/users
index 58563cd..f589298 160000
--- a/users
+++ b/users
@@ -1 +1 @@
-Subproject commit 58563cdc022363375c00ac580672f7daa22f6453
+Subproject commit f58929826f3dd8087afbfa3e8f2ac6e776267d40