From: Kai Moritz Date: Sun, 11 Jun 2023 07:40:54 +0000 (+0200) Subject: Docker-Setup in das Unterverzeichnis `docker` verschoben X-Git-Tag: producer/rest-producer--headers---2023-06-signal~8 X-Git-Url: https://juplo.de/gitweb/?a=commitdiff_plain;h=7b3a6be817b9d417f4c36cc9ea36bfa5a1bda343;p=demos%2Fkafka%2Ftraining Docker-Setup in das Unterverzeichnis `docker` verschoben --- diff --git a/README.sh b/README.sh index 76ed9c3..fae0348 100755 --- a/README.sh +++ b/README.sh @@ -4,13 +4,13 @@ IMAGE=juplo/rest-producer:1.0-SNAPSHOT if [ "$1" = "cleanup" ] then - docker-compose down -v --remove-orphans + docker-compose -f docker/docker-compose.yml down -v --remove-orphans mvn clean exit fi -docker-compose up -d kafka-1 kafka-2 kafka-3 cli -docker-compose rm -svf producer producer-0 producer-1 +docker-compose -f docker/docker-compose.yml up -d kafka-1 kafka-2 kafka-3 cli +docker-compose -f docker/docker-compose.yml rm -svf producer producer-0 producer-1 if [[ $(docker image ls -q $IMAGE) == "" || @@ -23,11 +23,11 @@ else docker image ls $IMAGE fi -docker-compose up setup -docker-compose up -d producer-0 producer-1 +docker-compose -f docker/docker-compose.yml up setup +docker-compose -f docker/docker-compose.yml up -d producer-0 producer-1 while ! [[ $(http 0:8000/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for producer-0..."; sleep 1; done while ! [[ $(http 0:8001/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for producer-1..."; sleep 1; done -docker-compose up -d consumer +docker-compose -f docker/docker-compose.yml up -d consumer # tag::http[] echo -n Nachricht 1 an Producer 0 | http -v :8000/foo @@ -36,9 +36,9 @@ echo -n Nachricht 1 an Producer 1 | http -v :8001/foo echo -n Nachricht 2 an Producer 0 | http -v :8000/bar echo -n Nachricht 2 an Producer 1 | http -v :8001/bar -docker-compose logs consumer +docker-compose -f docker/docker-compose.yml logs consumer -docker-compose exec -T cli bash << 'EOF' +docker-compose -f docker/docker-compose.yml exec -T cli bash << 'EOF' echo "Altering number of partitions from 3 to 7..." kafka-topics --bootstrap-server kafka:9092 --describe --topic test # tag::repartitioning[] @@ -47,7 +47,7 @@ kafka-topics --bootstrap-server kafka:9092 --alter --topic test --partitions 7 kafka-topics --bootstrap-server kafka:9092 --describe --topic test EOF -docker-compose restart producer-0 producer-1 +docker-compose -f docker/docker-compose.yml restart producer-0 producer-1 while ! [[ $(http 0:8000/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for producer-0..."; sleep 1; done while ! [[ $(http 0:8001/actuator/health 2> /dev/null) =~ "UP" ]]; do echo "Waiting for producer-1..."; sleep 1; done @@ -56,7 +56,7 @@ echo -n Nachricht 3 an Producer 1 | http -v :8001/foo echo -n Nachricht 4 an Producer 0 | http -v :8000/bar echo -n Nachricht 4 an Producer 1 | http -v :8001/bar -docker-compose logs consumer +docker-compose -f docker/docker-compose.yml logs consumer # tag::kafkacat[] kafkacat -b :9092 -t test -o 0 -p0 -f'p=%p|o=%o|k=%k|v=%s\n' -qe diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index baf0d31..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,143 +0,0 @@ -version: '3.2' -services: - kafka-0: - image: bitnami/kafka:3.3.1 - environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_PROCESS_ROLES: controller - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: CONTROLLER://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS: " " - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT - KAFKA_BROKER_ID: 0 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" - - kafka-1: - image: bitnami/kafka:3.3.1 - environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_PROCESS_ROLES: broker - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081 - KAFKA_BROKER_ID: 1 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" - ports: - - 9081:9081 - depends_on: - - kafka-0 - - kafka-2: - image: bitnami/kafka:3.3.1 - environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_PROCESS_ROLES: broker - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082 - KAFKA_BROKER_ID: 2 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" - ports: - - 9092:9082 - - 9082:9082 - networks: - default: - aliases: - - kafka - depends_on: - - kafka-0 - - kafka-3: - image: bitnami/kafka:3.3.1 - environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_PROCESS_ROLES: broker - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083 - KAFKA_BROKER_ID: 3 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" - ports: - - 9083:9083 - depends_on: - - kafka-0 - - - setup: - image: juplo/toolbox - command: > - bash -c " - kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test - kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 2 --replication-factor 3 --config min.insync.replicas=2 - kafka-topics --bootstrap-server kafka:9092 --describe --topic test - " - depends_on: - - kafka-1 - - kafka-2 - - kafka-3 - - cli: - image: juplo/toolbox - command: sleep infinity - depends_on: - - kafka-1 - - kafka-2 - - kafka-3 - - producer: - image: juplo/rest-producer:1.0-SNAPSHOT - ports: - - 8080:8080 - environment: - server.port: 8080 - producer.bootstrap-server: kafka:9092 - producer.client-id: producer - producer.topic: test - - producer-0: - image: juplo/rest-producer:1.0-SNAPSHOT - ports: - - 8000:8080 - environment: - server.port: 8080 - producer.bootstrap-server: kafka:9092 - producer.client-id: producer-0 - producer.topic: test - producer.partition: 0 - - producer-1: - image: juplo/rest-producer:1.0-SNAPSHOT - ports: - - 8001:8080 - environment: - server.port: 8080 - producer.bootstrap-server: kafka:9092 - producer.client-id: producer-1 - producer.topic: test - producer.partition: 1 - - consumer: - image: juplo/simple-consumer:1.0-SNAPSHOT - command: kafka:9092 test my-group consumer diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..baf0d31 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,143 @@ +version: '3.2' +services: + kafka-0: + image: bitnami/kafka:3.3.1 + environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: controller + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: CONTROLLER://:9092 + KAFKA_CFG_ADVERTISED_LISTENERS: " " + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT + KAFKA_BROKER_ID: 0 + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + + kafka-1: + image: bitnami/kafka:3.3.1 + environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: broker + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9081 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081 + KAFKA_BROKER_ID: 1 + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + ports: + - 9081:9081 + depends_on: + - kafka-0 + + kafka-2: + image: bitnami/kafka:3.3.1 + environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: broker + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9082 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082 + KAFKA_BROKER_ID: 2 + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + ports: + - 9092:9082 + - 9082:9082 + networks: + default: + aliases: + - kafka + depends_on: + - kafka-0 + + kafka-3: + image: bitnami/kafka:3.3.1 + environment: + KAFKA_ENABLE_KRAFT: 'yes' + KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw + KAFKA_CFG_PROCESS_ROLES: broker + KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_CFG_LISTENERS: BROKER://:9092, LOCALHOST://:9083 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083 + KAFKA_BROKER_ID: 3 + KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 0@kafka-0:9092 + ALLOW_PLAINTEXT_LISTENER: 'yes' + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + ports: + - 9083:9083 + depends_on: + - kafka-0 + + + setup: + image: juplo/toolbox + command: > + bash -c " + kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test + kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 2 --replication-factor 3 --config min.insync.replicas=2 + kafka-topics --bootstrap-server kafka:9092 --describe --topic test + " + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + + cli: + image: juplo/toolbox + command: sleep infinity + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + + producer: + image: juplo/rest-producer:1.0-SNAPSHOT + ports: + - 8080:8080 + environment: + server.port: 8080 + producer.bootstrap-server: kafka:9092 + producer.client-id: producer + producer.topic: test + + producer-0: + image: juplo/rest-producer:1.0-SNAPSHOT + ports: + - 8000:8080 + environment: + server.port: 8080 + producer.bootstrap-server: kafka:9092 + producer.client-id: producer-0 + producer.topic: test + producer.partition: 0 + + producer-1: + image: juplo/rest-producer:1.0-SNAPSHOT + ports: + - 8001:8080 + environment: + server.port: 8080 + producer.bootstrap-server: kafka:9092 + producer.client-id: producer-1 + producer.topic: test + producer.partition: 1 + + consumer: + image: juplo/simple-consumer:1.0-SNAPSHOT + command: kafka:9092 test my-group consumer