From f324379723b2f3d681230f571f82bfbb0d53cb75 Mon Sep 17 00:00:00 2001 From: Kai Moritz Date: Wed, 2 Nov 2022 18:50:19 +0100 Subject: [PATCH] WIP --- README.sh | 80 ++++--------------- docker-compose.yml | 195 ++++++++++++++------------------------------- pom.xml | 10 +-- 3 files changed, 76 insertions(+), 209 deletions(-) diff --git a/README.sh b/README.sh index 07e36d7..1394020 100755 --- a/README.sh +++ b/README.sh @@ -1,6 +1,6 @@ #!/bin/bash -IMAGE=juplo/sumup-adder-springified:1.0-SNAPSHOT +IMAGE=juplo/spring-consumer:1.0-SNAPSHOT if [ "$1" = "cleanup" ] then @@ -9,17 +9,16 @@ then exit fi -docker-compose rm -svf adder-1 adder-2 -docker-compose rm -svf mongo -docker-compose up -d zookeeper kafka-1 kafka-2 kafka-3 cli mongo express +trap 'kill $(jobs -p) 2>/dev/null' EXIT + +docker-compose up -d kafka-0 kafka-1 kafka-2 kafka-3 cli if [[ $(docker image ls -q $IMAGE) == "" || "$1" = "build" ]] then - docker-compose rm -svf adder-1 adder-2 - mvn -D skipTests clean install || exit + mvn clean install || exit else echo "Using image existing images:" docker image ls $IMAGE @@ -28,63 +27,12 @@ fi echo "Waiting for the Kafka-Cluster to become ready..." docker-compose exec cli cub kafka-ready -b kafka:9092 1 60 > /dev/null 2>&1 || exit 1 docker-compose up setup -docker-compose up -d gateway requests-1 requests-2 - -while ! [[ $(http 0:8080/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for gateway..."; sleep 1; done -while ! [[ $(http 0:8081/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for requests-1..."; sleep 1; done -while ! [[ $(http 0:8082/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for requests-2..."; sleep 1; done - -docker-compose up -d peter klaus - -docker-compose up -d adder-1 -while ! [[ $(http 0:8091/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for adder-1..."; sleep 1; done -while [[ "$(http :8091/results | jq -r .)" == "{}" ]]; do echo "Waiting for some results to show up on adder-1..."; sleep 1; done -http -v --pretty none -S :8091/results -echo - -sleep 3 -echo "Resultate für adder-1" -http -v --pretty none -S :8091/results -echo - -echo "Resultate für peter von adder-1" -http :8091/results/peter | jq .[].sum | uniq -echo "Resultate für klaus von adder-1" -http :8091/results/klaus | jq .[].sum | uniq - - -docker-compose up -d adder-2 -while ! [[ $(http 0:8092/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for adder-2..."; sleep 1; done -while [[ "$(http :8092/results | jq -r .)" == "{}" ]]; do echo "Waiting for some results to show up on adder-2..."; sleep 1; done -http -v --pretty none -S :8092/results -echo - -sleep 3 -echo "Resultate für adder-2" -http -v --pretty none -S :8092/results -echo - -echo "Resultate für peter von adder-1" -http :8091/results/peter | jq .[].sum | uniq -echo "Resultate für klaus von adder-1" -http :8091/results/klaus | jq .[].sum | uniq - -echo "Resultate für peter von adder-2" -http :8092/results/peter | jq .[].sum | uniq -echo "Resultate für klaus von adder-2" -http :8092/results/klaus | jq .[].sum | uniq - -docker-compose stop adder-1 -until [ $(http --check-status :8092/results/peter 2> /dev/null) ]; do echo "Waiting for some results for peter to show up on adder-2..."; sleep 1; done -until [ $(http --check-status :8092/results/klaus 2> /dev/null) ]; do echo "Waiting for some results for peter to show up on adder-2..."; sleep 1; done - -echo "Resultate für adder-2" -http -v --pretty none -S :8092/results -echo - -echo "Resultate für peter von adder-2" -http :8092/results/peter | jq .[].sum | uniq -echo "Resultate für klaus von adder-2" -http :8092/results/klaus | jq .[].sum | uniq - -docker-compose kill -s 9 peter klaus +docker-compose up -d producer + +mvn spring-boot:run & +sleep 10 +kill $(jobs -p) +mvn spring-boot:run & +sleep 10 +docker-compose stop producer +kill $(jobs -p) diff --git a/docker-compose.yml b/docker-compose.yml index 16fec5b..6d25cc1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,39 +1,56 @@ version: '3.2' services: - zookeeper: - image: confluentinc/cp-zookeeper:7.1.3 + kafka-0: + image: bitnami/kafka:3.3.1 environment: - ZOOKEEPER_CLIENT_PORT: 2181 - ports: - - 2181:2181 + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: controller + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: CONTROLLER://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: " " + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT + KAFKA_BROKER_ID: 0 + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" kafka-1: - image: confluentinc/cp-kafka:7.1.3 + image: bitnami/kafka:3.3.1 environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: broker + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081 KAFKA_BROKER_ID: 1 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: DOCKER://:9092, LOCALHOST://:9081 - KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka-1:9092, LOCALHOST://localhost:9081 - KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" ports: - 9081:9081 - depends_on: - - zookeeper kafka-2: - image: confluentinc/cp-kafka:7.1.3 + image: bitnami/kafka:3.3.1 environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: broker + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082 KAFKA_BROKER_ID: 2 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: DOCKER://:9092, LOCALHOST://:9082 - KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka-2:9092, LOCALHOST://localhost:9082 - KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" ports: - 9092:9082 - 9082:9082 @@ -41,133 +58,39 @@ services: default: aliases: - kafka - depends_on: - - zookeeper kafka-3: - image: confluentinc/cp-kafka:7.1.3 + image: bitnami/kafka:3.3.1 environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: broker + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083 KAFKA_BROKER_ID: 3 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: DOCKER://:9092, LOCALHOST://:9083 - KAFKA_ADVERTISED_LISTENERS: DOCKER://kafka-3:9092, LOCALHOST://localhost:9083 - KAFKA_INTER_BROKER_LISTENER_NAME: DOCKER - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: DOCKER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" ports: - 9083:9083 - depends_on: - - zookeeper - - mongo: - image: mongo:4.4.13 - ports: - - 27017:27017 - environment: - MONGO_INITDB_ROOT_USERNAME: juplo - MONGO_INITDB_ROOT_PASSWORD: training - - express: - image: mongo-express - ports: - - 8090:8081 - environment: - ME_CONFIG_MONGODB_ADMINUSERNAME: juplo - ME_CONFIG_MONGODB_ADMINPASSWORD: training - ME_CONFIG_MONGODB_URL: mongodb://juplo:training@mongo:27017/ - depends_on: - - mongo setup: image: juplo/toolbox command: > bash -c " - kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic in - kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic out - kafka-topics --bootstrap-server kafka:9092 --create --topic in --partitions 2 --replication-factor 3 --config min.insync.replicas=2 - kafka-topics --bootstrap-server kafka:9092 --create --topic out --partitions 2 --replication-factor 3 --config min.insync.replicas=2 - kafka-topics --bootstrap-server kafka:9092 --describe --topic in - kafka-topics --bootstrap-server kafka:9092 --describe --topic out + kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test + kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 2 --replication-factor 3 --config min.insync.replicas=2 + kafka-topics --bootstrap-server kafka:9092 --describe --topic test " cli: image: juplo/toolbox command: sleep infinity - gateway: - image: juplo/sumup-gateway--springified:1.0-SNAPSHOT - ports: - - 8080:8080 - environment: - server.port: 8080 - spring.kafka.bootstrap-servers: kafka:9092 - spring.kafka.client-id: gateway - sumup.gateway.topic: in - - requests-1: - image: juplo/sumup-requests-json:1.0-SNAPSHOT - ports: - - 8081:8080 - environment: - server.port: 8080 - sumup.requests.bootstrap-server: kafka:9092 - sumup.requests.client-id: requests-1 - - requests-2: - image: juplo/sumup-requests-json:1.0-SNAPSHOT - ports: - - 8082:8080 - environment: - server.port: 8080 - sumup.requests.bootstrap-server: kafka:9092 - sumup.requests.client-id: requests-2 - - adder-1: - image: juplo/sumup-adder-springified:1.0-SNAPSHOT - ports: - - 8091:8080 - environment: - server.port: 8080 - spring.kafka.bootstrap-servers: kafka:9092 - spring.kafak.client-id: adder-1 - spring.kafka.auto-commit-interval: 1s - sumup.adder.throttle: 3ms - spring.data.mongodb.uri: mongodb://juplo:training@mongo:27017 - spring.data.mongodb.database: juplo - logging.level.org.apache.kafka.clients.consumer: DEBUG - - adder-2: - image: juplo/sumup-adder-springified:1.0-SNAPSHOT - ports: - - 8092:8080 - environment: - server.port: 8080 - spring.kafka.bootstrap-servers: kafka:9092 - spring.kafak.client-id: adder-2 - spring.kafka.auto-commit-interval: 1s - sumup.adder.throttle: 3ms - spring.data.mongodb.uri: mongodb://juplo:training@mongo:27017 - spring.data.mongodb.database: juplo - logging.level.org.apache.kafka.clients.consumer: DEBUG - - peter: - image: juplo/toolbox - command: > - bash -c " - while [[ true ]]; - do - echo 666 | http -v gateway:8080/peter; - sleep 1; - done - " - klaus: - image: juplo/toolbox - command: > - bash -c " - while [[ true ]]; - do - echo 666 | http -v gateway:8080/klaus; - sleep 1; - done - " + producer: + image: juplo/simple-producer:1.0-SNAPSHOT + command: producer diff --git a/pom.xml b/pom.xml index a252d1c..9935ed8 100644 --- a/pom.xml +++ b/pom.xml @@ -12,10 +12,10 @@ de.juplo.kafka - sumup-adder-springified + spring-consumer 1.0-SNAPSHOT - SumUp Adder - Calculates the sum for the send messages. This version consumes JSON-messages. + Spring Consumer + Super Simple Consumer-Group, that is configured by Spring Kafka 11 @@ -26,10 +26,6 @@ org.springframework.boot spring-boot-starter-web - - org.springframework.boot - spring-boot-starter-data-mongodb - org.springframework.boot spring-boot-starter-validation -- 2.20.1