Cleanup setzt GIT-Repo von Cruise Control zurück
[demos/kafka/training] / docker / docker-compose.yml
1 version: '3.2'
2 services:
3   zookeeper:
4     image: confluentinc/cp-zookeeper:7.5.1
5     environment:
6       ZOOKEEPER_CLIENT_PORT: 2181
7     ports:
8       - 2181:2181
9     volumes:
10       - zookeeper-data:/var/lib/zookeeper/data
11       - zookeeper-log:/var/lib/zookeeper/log
12
13   kafka-1:
14     image: confluentinc/cp-kafka:7.5.1
15     environment:
16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
20       KAFKA_BROKER_ID: 1
21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
26       KAFKA_SUPER_USERS: User:ANONYMOUS
27       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
28       KAFKA_OPTS:
29         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
30         -Dcom.sun.management.jmxremote=true
31         -Dcom.sun.management.jmxremote.port=9101
32         -Dcom.sun.management.jmxremote.authenticate=false
33       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
34       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
35       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
36       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
37       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
38       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
39       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
40       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
41     volumes:
42       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
43       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
44       - kafka-1-data:/var/lib/kafka/data
45     ports:
46       - 9081:9081
47     stop_grace_period: 120s
48     depends_on:
49       - zookeeper
50
51   kafka-2:
52     image: confluentinc/cp-kafka:7.5.1
53     environment:
54       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
55       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
56       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
57       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
58       KAFKA_BROKER_ID: 2
59       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
60       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
61       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
62       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
63       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
64       KAFKA_SUPER_USERS: User:ANONYMOUS
65       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
66       KAFKA_OPTS:
67         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
68         -Dcom.sun.management.jmxremote=true
69         -Dcom.sun.management.jmxremote.port=9101
70         -Dcom.sun.management.jmxremote.authenticate=false
71       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
72       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
73       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
74       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
75       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
76       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
77       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
78       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
79     volumes:
80       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
81       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
82       - kafka-2-data:/var/lib/kafka/data
83     ports:
84       - 9092:9082
85       - 9082:9082
86     networks:
87       default:
88         aliases:
89           - kafka
90     stop_grace_period: 120s
91     depends_on:
92       - zookeeper
93
94   kafka-3:
95     image: confluentinc/cp-kafka:7.5.1
96     environment:
97       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
98       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
99       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
100       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
101       KAFKA_BROKER_ID: 3
102       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
103       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
104       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
105       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
106       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
107       KAFKA_SUPER_USERS: User:ANONYMOUS
108       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
109       KAFKA_OPTS:
110         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
111         -Dcom.sun.management.jmxremote=true
112         -Dcom.sun.management.jmxremote.port=9101
113         -Dcom.sun.management.jmxremote.authenticate=false
114       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
115       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
116       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
117       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
118       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
119       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
120       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
121       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
122     volumes:
123       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
124       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
125       - kafka-3-data:/var/lib/kafka/data
126     ports:
127       - 9083:9083
128     stop_grace_period: 120s
129     depends_on:
130       - zookeeper
131
132   kafka-4:
133     image: confluentinc/cp-kafka:7.5.1
134     environment:
135       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
136       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
137       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
138       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
139       KAFKA_BROKER_ID: 4
140       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
141       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
142       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
143       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
144       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
145       KAFKA_SUPER_USERS: User:ANONYMOUS
146       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
147       KAFKA_OPTS:
148         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
149         -Dcom.sun.management.jmxremote=true
150         -Dcom.sun.management.jmxremote.port=9101
151         -Dcom.sun.management.jmxremote.authenticate=false
152       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
153       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
154       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
155       KAFKA_LISTENER_NAME_DOCKER_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
156       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: SCRAM-SHA-512
157       KAFKA_LISTENER_NAME_LOCALHOST_SCRAM-SHA-512_SASL_JAAS_CONFIG: org.apache.kafka.common.security.scram.ScramLoginModule required;
158       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
159       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
160     volumes:
161       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
162       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
163       - kafka-4-data:/var/lib/kafka/data
164     ports:
165       - 9084:9084
166     stop_grace_period: 120s
167     depends_on:
168       - zookeeper
169
170   cruise-control:
171     image: juplo/cruise-control
172     environment:
173       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
174       ZOOKEEPER_CONNECT: zookeeper:2181/
175     ports:
176       - "9090:9090"
177
178   schema-registry:
179     image: confluentinc/cp-schema-registry:7.5.1
180     environment:
181       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
182       SCHEMA_REGISTRY_HOST_NAME: schema-registry
183       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
184       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
185       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
186       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: SCRAM-SHA-512
187       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
188         org.apache.kafka.common.security.scram.ScramLoginModule required
189         username="schemaregistry"
190         password="schemaregistry-secret";
191     ports:
192       - 8085:8085
193     depends_on:
194       - kafka-1
195       - kafka-2
196       - kafka-3
197
198   connect:
199     image: confluentinc/cp-kafka-connect:7.5.1
200     environment:
201       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
202       CONNECT_REST_PORT: 8083
203       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
204       CONNECT_REST_ADVERTISED_HOST_NAME: connect
205       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
206       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
207       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
208       CONNECT_GROUP_ID: kafka-connect
209       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
210       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
211       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
212       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
213       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
214       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
215       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
216       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
217       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
218       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
219       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
220       CONNECT_PLUGIN_PATH: /usr/share/java/
221       # Configure the Connect workers to use SASL/PLAIN.
222       CONNECT_SASL_MECHANISM: SCRAM-SHA-512
223       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
224       # JAAS
225       CONNECT_SASL_JAAS_CONFIG: >
226         org.apache.kafka.common.security.scram.ScramLoginModule required
227         username="connect"
228         password="connect-secret";
229       # producer
230       CONNECT_PRODUCER_SASL_MECHANISM: SCRAM-SHA-512
231       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
232       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
233         org.apache.kafka.common.security.scram.ScramLoginModule required
234         username="connect"
235         password="connect-secret";
236       # consumer
237       CONNECT_CONSUMER_SASL_MECHANISM: SCRAM-SHA-512
238       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
239       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
240         org.apache.kafka.common.security.scram.ScramLoginModule required
241         username="connect"
242         password="connect-secret";
243
244     ports:
245       - 8083:8083
246     depends_on:
247       - schema-registry
248
249   cli:
250     image: juplo/toolbox
251     command: sleep infinity
252     stop_grace_period: 0s
253     depends_on:
254       - kafka-1
255       - kafka-2
256       - kafka-3
257
258   setup:
259     image: juplo/toolbox
260     command:
261       - bash
262       - -c
263       - |
264         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
265         if [ -e INITIALIZED ]
266         then
267           echo -n Bereits konfiguriert: 
268           cat INITIALIZED
269           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
270           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
271         else
272           kafka-topics --bootstrap-server kafka:9091 \
273                        --delete \
274                        --if-exists \
275                        --topic test
276           kafka-topics --bootstrap-server kafka:9091 \
277                        --create \
278                        --topic test \
279                        --partitions 2 \
280                        --replication-factor 3 \
281                        --config min.insync.replicas=2 \
282           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
283           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
284           kafka-topics --bootstrap-server kafka:9091 \
285                        --delete \
286                        --if-exists \
287                        --topic __CruiseControlMetrics
288           kafka-topics --bootstrap-server kafka:9091 \
289                        --create \
290                        --topic __CruiseControlMetrics \
291                        --partitions 2 \
292                        --replication-factor 3 \
293                        --config min.insync.replicas=2 \
294           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
295           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
296         fi \
297         && date > INITIALIZED
298     stop_grace_period: 0s
299     depends_on:
300       - cli
301
302   zoonavigator:
303     image: elkozmon/zoonavigator:1.1.2
304     ports:
305       - "8000:80"
306     environment:
307       HTTP_PORT: 80
308       CONNECTION_JUPLO_NAME: juplo
309       CONNECTION_JUPLO_CONN: zookeeper:2181
310       AUTO_CONNECT_CONNECTION_ID: JUPLO
311     depends_on:
312       - zookeeper
313
314   akhq:
315     image: tchiotludo/akhq:0.23.0
316     ports:
317       - 8888:8080
318     environment:
319       AKHQ_CONFIGURATION: |
320         akhq:
321           connections:
322             docker-kafka-server:
323               properties:
324                 bootstrap.servers: "kafka:9092"
325                 security.protocol: SASL_PLAINTEXT
326                 sasl.mechanism: SCRAM-SHA-512
327                 sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="ui" password="ui-secret";
328               schema-registry:
329                 url: "http://schema-registry:8085"
330               connect:
331                 - name: "connect"
332                   url: "http://connect:8083"
333     depends_on:
334       - kafka-1
335       - kafka-2
336       - kafka-3
337
338   producer:
339     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
340     environment:
341       producer.throttle-ms: 1
342       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
343       spring.kafka.client-id: producer
344       spring.kafka.template.default-topic: test
345       spring.kafka.producer.properties.linger.ms: 100
346       spring.kafka.jaas.enabled: "true"
347       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
348       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
349       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
350
351   consumer-1:
352     image: juplo/spring-consumer:1.0-SNAPSHOT
353     environment:
354       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
355       spring.kafka.client-id: consumer-1
356       spring.kafka.jaas.enabled: "true"
357       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
358       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
359       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
360
361   consumer-2:
362     image: juplo/spring-consumer:1.0-SNAPSHOT
363     environment:
364       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
365       spring.kafka.client-id: consumer-1
366       spring.kafka.jaas.enabled: "true"
367       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
368       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
369       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
370
371   consumer-3:
372     image: juplo/spring-consumer:1.0-SNAPSHOT
373     environment:
374       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
375       spring.kafka.client-id: consumer-1
376       spring.kafka.jaas.enabled: "true"
377       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
378       spring.kafka.properties.sasl.mechanism: SCRAM-SHA-512
379       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.scram.ScramLoginModule required username="client" password="client-secret";
380
381 volumes:
382   zookeeper-data:
383   zookeeper-log:
384   kafka-1-data:
385   kafka-2-data:
386   kafka-3-data:
387   kafka-4-data: