Die Adressen DOCKER und LOCALHOST über ACLs abgesichert
[demos/kafka/training] / docker / docker-compose.yml
1 version: '3.2'
2 services:
3   zookeeper:
4     image: confluentinc/cp-zookeeper:7.5.1
5     environment:
6       ZOOKEEPER_CLIENT_PORT: 2181
7     ports:
8       - 2181:2181
9     volumes:
10       - zookeeper-data:/var/lib/zookeeper/data
11       - zookeeper-log:/var/lib/zookeeper/log
12
13   kafka-1:
14     image: confluentinc/cp-kafka:7.5.1
15     environment:
16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
17       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081
18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9091, DOCKER://kafka-1:9092, LOCALHOST://localhost:9081
20       KAFKA_BROKER_ID: 1
21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
26       KAFKA_SUPER_USERS: User:ANONYMOUS
27       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
28       KAFKA_OPTS:
29         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
30         -Dcom.sun.management.jmxremote=true
31         -Dcom.sun.management.jmxremote.port=9101
32         -Dcom.sun.management.jmxremote.authenticate=false
33       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
34       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
35       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
36       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
37         org.apache.kafka.common.security.plain.PlainLoginModule required \
38         user_schemaregistry="schemaregistry-secret" \
39         user_connect="connect-secret" \
40         user_ui="ui-secret" \
41         user_client="client-secret";
42       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
43       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
44         org.apache.kafka.common.security.plain.PlainLoginModule required \
45         user_client="client-secret";
46       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
47       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
48     volumes:
49       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
50       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
51       - kafka-1-data:/var/lib/kafka/data
52     ports:
53       - 9081:9081
54     stop_grace_period: 120s
55     depends_on:
56       - zookeeper
57
58   kafka-2:
59     image: confluentinc/cp-kafka:7.5.1
60     environment:
61       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
62       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082
63       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
64       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9091, DOCKER://kafka-2:9092, LOCALHOST://localhost:9082
65       KAFKA_BROKER_ID: 2
66       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
67       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
68       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
69       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
70       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
71       KAFKA_SUPER_USERS: User:ANONYMOUS
72       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
73       KAFKA_OPTS:
74         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
75         -Dcom.sun.management.jmxremote=true
76         -Dcom.sun.management.jmxremote.port=9101
77         -Dcom.sun.management.jmxremote.authenticate=false
78       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
79       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
80       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
81       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
82         org.apache.kafka.common.security.plain.PlainLoginModule required \
83         user_schemaregistry="schemaregistry-secret" \
84         user_connect="connect-secret" \
85         user_ui="ui-secret" \
86         user_client="client-secret";
87       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
88       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
89         org.apache.kafka.common.security.plain.PlainLoginModule required \
90         user_client="client-secret";
91       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
92       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
93     volumes:
94       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
95       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
96       - kafka-2-data:/var/lib/kafka/data
97     ports:
98       - 9092:9082
99       - 9082:9082
100     networks:
101       default:
102         aliases:
103           - kafka
104     stop_grace_period: 120s
105     depends_on:
106       - zookeeper
107
108   kafka-3:
109     image: confluentinc/cp-kafka:7.5.1
110     environment:
111       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
112       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083
113       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
114       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9091, DOCKER://kafka-3:9092, LOCALHOST://localhost:9083
115       KAFKA_BROKER_ID: 3
116       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
117       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
118       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
119       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
120       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
121       KAFKA_SUPER_USERS: User:ANONYMOUS
122       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
123       KAFKA_OPTS:
124         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
125         -Dcom.sun.management.jmxremote=true
126         -Dcom.sun.management.jmxremote.port=9101
127         -Dcom.sun.management.jmxremote.authenticate=false
128       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
129       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
130       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
131       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
132         org.apache.kafka.common.security.plain.PlainLoginModule required \
133         user_schemaregistry="schemaregistry-secret" \
134         user_connect="connect-secret" \
135         user_ui="ui-secret" \
136         user_client="client-secret";
137       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
138       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
139         org.apache.kafka.common.security.plain.PlainLoginModule required \
140         user_client="client-secret";
141       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
142       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
143     volumes:
144       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
145       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
146       - kafka-3-data:/var/lib/kafka/data
147     ports:
148       - 9083:9083
149     stop_grace_period: 120s
150     depends_on:
151       - zookeeper
152
153   kafka-4:
154     image: confluentinc/cp-kafka:7.5.1
155     environment:
156       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
157       KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084
158       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT
159       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084
160       KAFKA_BROKER_ID: 4
161       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
162       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
163       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
164       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
165       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false"
166       KAFKA_SUPER_USERS: User:ANONYMOUS
167       KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO
168       KAFKA_OPTS:
169         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
170         -Dcom.sun.management.jmxremote=true
171         -Dcom.sun.management.jmxremote.port=9101
172         -Dcom.sun.management.jmxremote.authenticate=false
173       KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter
174       KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091
175       KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN
176       KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: |
177         org.apache.kafka.common.security.plain.PlainLoginModule required \
178         user_schemaregistry="schemaregistry-secret" \
179         user_connect="connect-secret" \
180         user_ui="ui-secret" \
181         user_client="client-secret";
182       KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN
183       KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: |
184         org.apache.kafka.common.security.plain.PlainLoginModule required \
185         user_client="client-secret";
186       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
187       KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512
188     volumes:
189       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
190       - ./cruise-control-metrics-reporter.jar:/usr/share/java/kafka/cruise-control-metrics-reporter.jar:ro
191       - kafka-4-data:/var/lib/kafka/data
192     ports:
193       - 9084:9084
194     stop_grace_period: 120s
195     depends_on:
196       - zookeeper
197
198   cruise-control:
199     image: juplo/cruise-control
200     environment:
201       BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091
202       ZOOKEEPER_CONNECT: zookeeper:2181/
203     ports:
204       - "9090:9090"
205
206   schema-registry:
207     image: confluentinc/cp-schema-registry:7.5.1
208     environment:
209       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
210       SCHEMA_REGISTRY_HOST_NAME: schema-registry
211       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
212       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
213       SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: SASL_PLAINTEXT
214       SCHEMA_REGISTRY_KAFKASTORE_SASL_MECHANISM: PLAIN
215       SCHEMA_REGISTRY_KAFKASTORE_SASL_JAAS_CONFIG: >
216         org.apache.kafka.common.security.plain.PlainLoginModule
217         required
218         username="schemaregistry"
219         password="schemaregistry-secret";
220     ports:
221       - 8085:8085
222     depends_on:
223       - kafka-1
224       - kafka-2
225       - kafka-3
226
227   connect:
228     image: confluentinc/cp-kafka-connect:7.5.1
229     environment:
230       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
231       CONNECT_REST_PORT: 8083
232       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
233       CONNECT_REST_ADVERTISED_HOST_NAME: connect
234       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
235       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
236       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
237       CONNECT_GROUP_ID: kafka-connect
238       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
239       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
240       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
241       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
242       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
243       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
244       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
245       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
246       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
247       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
248       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
249       CONNECT_PLUGIN_PATH: /usr/share/java/
250       # Configure the Connect workers to use SASL/PLAIN.
251       CONNECT_SASL_MECHANISM: PLAIN
252       CONNECT_SECURITY_PROTOCOL: SASL_PLAINTEXT
253       # JAAS
254       CONNECT_SASL_JAAS_CONFIG: >
255         org.apache.kafka.common.security.plain.PlainLoginModule required
256         username="connect"
257         password="connect-secret";
258       # producer
259       CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
260       CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_PLAINTEXT
261       CONNECT_PRODUCER_SASL_JAAS_CONFIG: >
262         org.apache.kafka.common.security.plain.PlainLoginModule required
263         username="connect"
264         password="connect-secret";
265       # consumer
266       CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
267       CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_PLAINTEXT
268       CONNECT_CONSUMER_SASL_JAAS_CONFIG: >
269         org.apache.kafka.common.security.plain.PlainLoginModule required
270         username="connect"
271         password="connect-secret";
272
273     ports:
274       - 8083:8083
275     depends_on:
276       - schema-registry
277
278   cli:
279     image: juplo/toolbox
280     command: sleep infinity
281     stop_grace_period: 0s
282     depends_on:
283       - kafka-1
284       - kafka-2
285       - kafka-3
286
287   setup:
288     image: juplo/toolbox
289     command:
290       - bash
291       - -c
292       - |
293         cub kafka-ready -b kafka-1:9091,kafka-2:9091,kafka-3:9091 3 60 > /dev/null 2>&1 || exit 1
294         if [ -e INITIALIZED ]
295         then
296           echo -n Bereits konfiguriert: 
297           cat INITIALIZED
298           kafka-topics --bootstrap-server kafka:9091 --describe --topic test
299           kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
300         else
301           kafka-topics --bootstrap-server kafka:9091 \
302                        --delete \
303                        --if-exists \
304                        --topic test
305           kafka-topics --bootstrap-server kafka:9091 \
306                        --create \
307                        --topic test \
308                        --partitions 2 \
309                        --replication-factor 3 \
310                        --config min.insync.replicas=2 \
311           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
312           && kafka-topics --bootstrap-server kafka:9091 --describe --topic test
313           kafka-topics --bootstrap-server kafka:9091 \
314                        --delete \
315                        --if-exists \
316                        --topic __CruiseControlMetrics
317           kafka-topics --bootstrap-server kafka:9091 \
318                        --create \
319                        --topic __CruiseControlMetrics \
320                        --partitions 2 \
321                        --replication-factor 3 \
322                        --config min.insync.replicas=2 \
323           && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \
324           && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics
325         fi \
326         && date > INITIALIZED
327     stop_grace_period: 0s
328     depends_on:
329       - cli
330
331   zoonavigator:
332     image: elkozmon/zoonavigator:1.1.2
333     ports:
334       - "8000:80"
335     environment:
336       HTTP_PORT: 80
337       CONNECTION_JUPLO_NAME: juplo
338       CONNECTION_JUPLO_CONN: zookeeper:2181
339       AUTO_CONNECT_CONNECTION_ID: JUPLO
340     depends_on:
341       - zookeeper
342
343   akhq:
344     image: tchiotludo/akhq:0.23.0
345     ports:
346       - 8888:8080
347     environment:
348       AKHQ_CONFIGURATION: |
349         akhq:
350           connections:
351             docker-kafka-server:
352               properties:
353                 bootstrap.servers: "kafka:9092"
354                 security.protocol: SASL_PLAINTEXT
355                 sasl.mechanism: PLAIN
356                 sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="ui" password="ui-secret";
357               schema-registry:
358                 url: "http://schema-registry:8085"
359               connect:
360                 - name: "connect"
361                   url: "http://connect:8083"
362     depends_on:
363       - kafka-1
364       - kafka-2
365       - kafka-3
366
367   producer:
368     image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT
369     environment:
370       producer.throttle-ms: 1
371       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
372       spring.kafka.client-id: producer
373       spring.kafka.template.default-topic: test
374       spring.kafka.producer.properties.linger.ms: 100
375       spring.kafka.jaas.enabled: "true"
376       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
377       spring.kafka.properties.sasl.mechanism: PLAIN
378       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
379
380   consumer-1:
381     image: juplo/spring-consumer:1.0-SNAPSHOT
382     environment:
383       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
384       spring.kafka.client-id: consumer-1
385       spring.kafka.jaas.enabled: "true"
386       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
387       spring.kafka.properties.sasl.mechanism: PLAIN
388       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
389
390   consumer-2:
391     image: juplo/spring-consumer:1.0-SNAPSHOT
392     environment:
393       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
394       spring.kafka.client-id: consumer-1
395       spring.kafka.jaas.enabled: "true"
396       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
397       spring.kafka.properties.sasl.mechanism: PLAIN
398       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
399
400   consumer-3:
401     image: juplo/spring-consumer:1.0-SNAPSHOT
402     environment:
403       spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092
404       spring.kafka.client-id: consumer-1
405       spring.kafka.jaas.enabled: "true"
406       spring.kafka.properties.security.protocol: SASL_PLAINTEXT
407       spring.kafka.properties.sasl.mechanism: PLAIN
408       spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret";
409
410 volumes:
411   zookeeper-data:
412   zookeeper-log:
413   kafka-1-data:
414   kafka-2-data:
415   kafka-3-data:
416   kafka-4-data: