juplo.de Git - demos/kafka/training/blob - docker/docker-compose.yml

   1 version: '3.2'
   2 services:
   3   zookeeper:
   4     image: confluentinc/cp-zookeeper:7.5.1
   5     environment:
   6       ZOOKEEPER_CLIENT_PORT: 2181
   7     ports:
   8       - 2181:2181
   9     volumes:
  10       - zookeeper-data:/var/lib/zookeeper/data
  11       - zookeeper-log:/var/lib/zookeeper/log
  12
  13   kafka-1:
  14     image: confluentinc/cp-kafka:7.5.1
  15     environment:
  16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
  18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
  20       KAFKA_BROKER_ID: 1
  21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  26       KAFKA_OPTS:
  27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  28         -Dcom.sun.management.jmxremote=true
  29         -Dcom.sun.management.jmxremote.port=9101
  30         -Dcom.sun.management.jmxremote.authenticate=false
  31       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  32     volumes:
  33       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  34       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  35       - kafka-1-data:/var/lib/kafka/data
  36     ports:
  37       - 9081:9081
  38     stop_grace_period: 120s
  39     depends_on:
  40       - zookeeper
  41
  42   kafka-2:
  43     image: confluentinc/cp-kafka:7.5.1
  44     environment:
  45       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  46       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
  47       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  48       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
  49       KAFKA_BROKER_ID: 2
  50       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  51       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  52       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  53       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  54       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  55       KAFKA_OPTS:
  56         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  57         -Dcom.sun.management.jmxremote=true
  58         -Dcom.sun.management.jmxremote.port=9101
  59         -Dcom.sun.management.jmxremote.authenticate=false
  60       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  61     volumes:
  62       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  63       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  64       - kafka-2-data:/var/lib/kafka/data
  65     ports:
  66       - 9092:9082
  67       - 9082:9082
  68     networks:
  69       default:
  70         aliases:
  71           - kafka
  72     stop_grace_period: 120s
  73     depends_on:
  74       - zookeeper
  75
  76   kafka-3:
  77     image: confluentinc/cp-kafka:7.5.1
  78     environment:
  79       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
  80       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
  81       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
  82       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
  83       KAFKA_BROKER_ID: 3
  84       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
  85       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
  86       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
  87       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
  88       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
  89       KAFKA_OPTS:
  90         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
  91         -Dcom.sun.management.jmxremote=true
  92         -Dcom.sun.management.jmxremote.port=9101
  93         -Dcom.sun.management.jmxremote.authenticate=false
  94       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
  95     volumes:
  96       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
  97       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
  98       - kafka-3-data:/var/lib/kafka/data
  99     ports:
 100       - 9083:9083
 101     stop_grace_period: 120s
 102     depends_on:
 103       - zookeeper
 104
 105   cruise-control:
 106     image: juplo/cruise-control
 107     environment:
 108       BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 109       ZOOKEEPER_CONNECT: zookeeper:2181/
 110       CORS_ORIGIN: http://localhost:9080
 111     ports:
 112       - "9090:9090"
 113
 114   schema-registry:
 115     image: confluentinc/cp-schema-registry:7.5.1
 116     environment:
 117       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 118       SCHEMA_REGISTRY_HOST_NAME: schema-registry
 119       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
 120       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
 121     ports:
 122       - 8085:8085
 123     depends_on:
 124       - kafka-1
 125       - kafka-2
 126       - kafka-3
 127
 128   connect:
 129     image: confluentinc/cp-kafka-connect:7.5.1
 130     environment:
 131       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
 132       CONNECT_REST_PORT: 8083
 133       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
 134       CONNECT_REST_ADVERTISED_HOST_NAME: connect
 135       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
 136       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
 137       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
 138       CONNECT_GROUP_ID: kafka-connect
 139       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
 140       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
 141       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 142       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
 143       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
 144       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
 145       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 146       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
 147       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
 148       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
 149       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
 150       CONNECT_PLUGIN_PATH: /usr/share/java/
 151     ports:
 152       - 8083:8083
 153     depends_on:
 154       - schema-registry
 155
 156   cli:
 157     image: juplo/toolbox
 158     command: sleep infinity
 159     stop_grace_period: 0s
 160     depends_on:
 161       - kafka-1
 162       - kafka-2
 163       - kafka-3
 164
 165   setup:
 166     image: juplo/toolbox
 167     command:
 168       - bash
 169       - -c
 170       - |
 171         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
 172         if [ -e INITIALIZED ]
 173         then
 174           echo -n Bereits konfiguriert:
 175           cat INITIALIZED
 176           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
 177           kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics
 178         else
 179           kafka-topics --bootstrap-server kafka:9092 \
 180                        --delete \
 181                        --if-exists \
 182                        --topic test
 183           kafka-topics --bootstrap-server kafka:9092 \
 184                        --delete \
 185                        --if-exists \
 186                        --topic __CruiseControlMetrics
 187           kafka-topics --bootstrap-server kafka:9092 \
 188                        --create \
 189                        --topic test \
 190                        --partitions 2 \
 191                        --replication-factor 3 \
 192                        --config min.insync.replicas=2 \
 193           kafka-topics --bootstrap-server kafka:9092 \
 194                        --create \
 195                        --topic __CruiseControlMetrics \
 196                        --partitions 2 \
 197                        --replication-factor 3 \
 198                        --config min.insync.replicas=2 \
 199           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
 200           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \
 201           && kafka-topics --bootstrap-server kafka:9092 --describe --topic __CruiseControlMetrics \
 202           && date > INITIALIZED
 203         fi
 204     stop_grace_period: 0s
 205     depends_on:
 206       - cli
 207
 208   zoonavigator:
 209     image: elkozmon/zoonavigator:1.1.2
 210     ports:
 211       - "8000:80"
 212     environment:
 213       HTTP_PORT: 80
 214       CONNECTION_JUPLO_NAME: juplo
 215       CONNECTION_JUPLO_CONN: zookeeper:2181
 216       AUTO_CONNECT_CONNECTION_ID: JUPLO
 217     depends_on:
 218       - zookeeper
 219
 220   akhq:
 221     image: tchiotludo/akhq:0.23.0
 222     ports:
 223       - 8888:8080
 224     environment:
 225       AKHQ_CONFIGURATION: |
 226         akhq:
 227           connections:
 228             docker-kafka-server:
 229               properties:
 230                 bootstrap.servers: "kafka:9092"
 231               schema-registry:
 232                 url: "http://schema-registry:8085"
 233               connect:
 234                 - name: "connect"
 235                   url: "http://connect:8083"
 236     depends_on:
 237       - kafka-1
 238       - kafka-2
 239       - kafka-3
 240
 241   cruise-control-ui:
 242     image: juplo/cruise-control-ui
 243     volumes:
 244       - ./config.csv:/usr/share/nginx/html/static/config.csv
 245     ports:
 246       - "9080:80"
 247
 248 volumes:
 249   zookeeper-data:
 250   zookeeper-log:
 251   kafka-1-data:
 252   kafka-2-data:
 253   kafka-3-data: