X-Git-Url: https://juplo.de/gitweb/?a=blobdiff_plain;f=docker%2Fdocker-compose.yml;h=df24c29206f60550d114c338184ea049df1c320f;hb=refs%2Fheads%2Fgrundlagen%2Fdocker-NEU;hp=aead4357654a392e7fbdc4ad6709e301c2a6eb88;hpb=059ff32e02e2b882388f3e4d4d332c60aba0a4b4;p=demos%2Fkafka%2Ftraining diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index aead435..df24c29 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,39 +1,54 @@ version: '3.2' services: + zookeeper: + image: confluentinc/cp-zookeeper:7.5.1 + environment: + ZOOKEEPER_CLIENT_PORT: 2181 + ports: + - 2181:2181 + volumes: + - zookeeper-data:/var/lib/zookeeper/data + - zookeeper-log:/var/lib/zookeeper/log kafka-1: - image: bitnami/kafka:3.4 + image: confluentinc/cp-kafka:7.5.1 environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: CONTROLLER://:9093, BROKER://:9092, LOCALHOST://:9081 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081 - KAFKA_CFG_NODE_ID: 1 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093, 2@kafka-2:9093, 3@kafka-3:9093 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081 + KAFKA_BROKER_ID: 1 + KAFKA_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_OPTS: + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=9101 + -Dcom.sun.management.jmxremote.authenticate=false + volumes: + - kafka-1-data:/var/lib/kafka/data ports: - 9081:9081 + depends_on: + - zookeeper kafka-2: - image: bitnami/kafka:3.4 + image: confluentinc/cp-kafka:7.5.1 environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: CONTROLLER://:9093, BROKER://:9092, LOCALHOST://:9082 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082 - KAFKA_CFG_NODE_ID: 2 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093, 2@kafka-2:9093, 3@kafka-3:9093 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082 + KAFKA_BROKER_ID: 2 + KAFKA_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_OPTS: + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=9101 + -Dcom.sun.management.jmxremote.authenticate=false + volumes: + - kafka-2-data:/var/lib/kafka/data ports: - 9092:9082 - 9082:9082 @@ -41,37 +56,95 @@ services: default: aliases: - kafka + depends_on: + - zookeeper kafka-3: - image: bitnami/kafka:3.4 + image: confluentinc/cp-kafka:7.5.1 environment: - KAFKA_ENABLE_KRAFT: 'yes' - KAFKA_KRAFT_CLUSTER_ID: r7dMBY60T16TrNCGeXniLw - KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_CFG_LISTENERS: CONTROLLER://:9093, BROKER://:9092, LOCALHOST://:9083 - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, CONTROLLER:PLAINTEXT, LOCALHOST:PLAINTEXT - KAFKA_CFG_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083 - KAFKA_CFG_NODE_ID: 3 - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@kafka-1:9093, 2@kafka-2:9093, 3@kafka-3:9093 - ALLOW_PLAINTEXT_LISTENER: 'yes' - KAFKA_CFG_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083 + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT + KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083 + KAFKA_BROKER_ID: 3 + KAFKA_INTER_BROKER_LISTENER_NAME: BROKER + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_OPTS: + -Dcom.sun.management.jmxremote=true + -Dcom.sun.management.jmxremote.port=9101 + -Dcom.sun.management.jmxremote.authenticate=false + volumes: + - kafka-3-data:/var/lib/kafka/data ports: - 9083:9083 + depends_on: + - zookeeper + + schema-registry: + image: confluentinc/cp-schema-registry:7.5.1 + environment: + SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'PLAINTEXT://kafka:9092' + SCHEMA_REGISTRY_HOST_NAME: 'schema-registry' + SCHEMA_REGISTRY_LISTENERS: 'http://0.0.0.0:8085' + SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: 'INFO' + depends_on: + - kafka-1 + - kafka-2 + - kafka-3 + connect: + image: confluentinc/cp-kafka-connect:7.5.1 + environment: + CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092' + CONNECT_REST_PORT: '8083' + CONNECT_REST_LISTENERS: 'http://0.0.0.0:8083' + CONNECT_REST_ADVERTISED_HOST_NAME: 'connect' + CONNECT_CONFIG_STORAGE_TOPIC: '__connect-config' + CONNECT_OFFSET_STORAGE_TOPIC: '__connect-offsets' + CONNECT_STATUS_STORAGE_TOPIC: '__connect-status' + CONNECT_GROUP_ID: 'kafka-connect' + CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_KEY_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: 'true' + CONNECT_VALUE_CONVERTER: 'io.confluent.connect.avro.AvroConverter' + CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: 'http://schema-registry:8085' + CONNECT_INTERNAL_KEY_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_INTERNAL_VALUE_CONVERTER: 'org.apache.kafka.connect.json.JsonConverter' + CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: '1' + CONNECT_PLUGIN_PATH: ' /usr/share/java/' + depends_on: + - schema-registry setup: image: juplo/toolbox - command: > - bash -c " - kafka-topics --bootstrap-server kafka:9092 --delete --if-exists --topic test - kafka-topics --bootstrap-server kafka:9092 --create --topic test --partitions 2 --replication-factor 3 --config min.insync.replicas=2 - echo Das Topic \'test\' wurde erfolgreich angelegt: - kafka-topics --bootstrap-server kafka:9092 --describe --topic test - echo \'docker-compose restart -t0 setup\' löscht das Topic und legt es neu an - sleep infinity - " + command: + - bash + - -c + - | + if [ -e INITIALIZED ] + then + echo -n Bereits konfiguriert: + cat INITIALIZED + kafka-topics --bootstrap-server kafka:9092 --describe --topic test + else + kafka-topics --bootstrap-server kafka:9092 \ + --delete \ + --if-exists \ + --topic test + kafka-topics --bootstrap-server kafka:9092 \ + --create \ + --topic test \ + --partitions 2 \ + --replication-factor 3 \ + --config min.insync.replicas=2 \ + && echo Das Topic \'test\' wurde erfolgreich angelegt: \ + && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \ + && date > INITIALIZED + fi depends_on: - kafka-1 - kafka-2 @@ -80,7 +153,7 @@ services: akhq: image: tchiotludo/akhq:0.23.0 ports: - - 8000:8080 + - 8888:8080 environment: AKHQ_CONFIGURATION: | akhq: @@ -88,6 +161,11 @@ services: docker-kafka-server: properties: bootstrap.servers: "kafka:9092" + schema-registry: + url: "http://schema-registry:8085" + connect: + - name: "connect" + url: "http://connect:8083" depends_on: - kafka-1 - kafka-2 @@ -98,3 +176,10 @@ services: command: sleep infinity depends_on: - setup + +volumes: + zookeeper-data: + zookeeper-log: + kafka-1-data: + kafka-2-data: + kafka-3-data: