c8c32d56d3918f3e54b0dcbf2f43085a52c85ad9
[demos/kafka/training] / docker / docker-compose.yml
1 version: '3.2'
2 services:
3   zookeeper:
4     image: confluentinc/cp-zookeeper:7.5.1
5     environment:
6       ZOOKEEPER_CLIENT_PORT: 2181
7     ports:
8       - 2181:2181
9     volumes:
10       - zookeeper-data:/var/lib/zookeeper/data
11       - zookeeper-log:/var/lib/zookeeper/log
12
13   kafka-1:
14     image: confluentinc/cp-kafka:7.5.1
15     environment:
16       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
17       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9081
18       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
19       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-1:9092, LOCALHOST://localhost:9081
20       KAFKA_BROKER_ID: 1
21       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
22       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
23       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
24       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
25       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
26       KAFKA_OPTS:
27         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
28         -Dcom.sun.management.jmxremote=true
29         -Dcom.sun.management.jmxremote.port=9101
30         -Dcom.sun.management.jmxremote.authenticate=false
31     volumes:
32       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
33       - kafka-1-data:/var/lib/kafka/data
34     ports:
35       - 9081:9081
36     stop_grace_period: 120s
37     depends_on:
38       - zookeeper
39
40   kafka-2:
41     image: confluentinc/cp-kafka:7.5.1
42     environment:
43       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
44       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9082
45       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
46       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-2:9092, LOCALHOST://localhost:9082
47       KAFKA_BROKER_ID: 2
48       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
49       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
50       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
51       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
52       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
53       KAFKA_OPTS:
54         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
55         -Dcom.sun.management.jmxremote=true
56         -Dcom.sun.management.jmxremote.port=9101
57         -Dcom.sun.management.jmxremote.authenticate=false
58     volumes:
59       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
60       - kafka-2-data:/var/lib/kafka/data
61     ports:
62       - 9092:9082
63       - 9082:9082
64     networks:
65       default:
66         aliases:
67           - kafka
68     stop_grace_period: 120s
69     depends_on:
70       - zookeeper
71
72   kafka-3:
73     image: confluentinc/cp-kafka:7.5.1
74     environment:
75       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
76       KAFKA_LISTENERS: BROKER://:9092, LOCALHOST://:9083
77       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, LOCALHOST:PLAINTEXT
78       KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-3:9092, LOCALHOST://localhost:9083
79       KAFKA_BROKER_ID: 3
80       KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
81       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3
82       KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
83       KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer
84       KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "true"
85       KAFKA_OPTS:
86         -javaagent:/usr/share/java/cp-base-new/jmx_prometheus_javaagent-0.18.0.jar=0.0.0.0:80:/etc/jmx-exporter.yml
87         -Dcom.sun.management.jmxremote=true
88         -Dcom.sun.management.jmxremote.port=9101
89         -Dcom.sun.management.jmxremote.authenticate=false
90     volumes:
91       - ./jmx-exporter.yml:/etc/jmx-exporter.yml:ro
92       - kafka-3-data:/var/lib/kafka/data
93     ports:
94       - 9083:9083
95     stop_grace_period: 120s
96     depends_on:
97       - zookeeper
98
99   schema-registry:
100     image: confluentinc/cp-schema-registry:7.5.1
101     environment:
102       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
103       SCHEMA_REGISTRY_HOST_NAME: schema-registry
104       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8085
105       SCHEMA_REGISTRY_LOG4J_ROOT_LOGLEVEL: INFO
106     ports:
107       - 8085:8085
108     depends_on:
109       - kafka-1
110       - kafka-2
111       - kafka-3
112
113   connect:
114     image: confluentinc/cp-kafka-connect:7.5.1
115     environment:
116       CONNECT_BOOTSTRAP_SERVERS: kafka-1:9092,kafka-2:9092,kafka-3:9092
117       CONNECT_REST_PORT: 8083
118       CONNECT_REST_LISTENERS: http://0.0.0.0:8083
119       CONNECT_REST_ADVERTISED_HOST_NAME: connect
120       CONNECT_CONFIG_STORAGE_TOPIC: __connect-config
121       CONNECT_OFFSET_STORAGE_TOPIC: __connect-offsets
122       CONNECT_STATUS_STORAGE_TOPIC: __connect-status
123       CONNECT_GROUP_ID: kafka-connect
124       CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE: "true"
125       CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
126       CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
127       CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
128       CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
129       CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8085
130       CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
131       CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
132       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
133       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
134       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
135       CONNECT_PLUGIN_PATH: /usr/share/java/
136     ports:
137       - 8083:8083
138     depends_on:
139       - schema-registry
140
141   cli:
142     image: juplo/toolbox
143     command:
144       - bash
145       - -c
146       - >-
147         cd /home/appuser;
148         cat movies.json | kafka-avro-console-producer
149         --bootstrap-server kafka:9092
150         --topic test
151         --property schema.registry.url=http://schema-registry:8085
152         --property value.schema="$$(cat movie.avsc)";
153         exec sleep infinity
154     volumes:
155       - ./movie.avsc:/home/appuser/movie.avsc:ro
156       - ./movies.json:/home/appuser/movies.json:ro
157     stop_grace_period: 0s
158     depends_on:
159       - kafka-1
160       - kafka-2
161       - kafka-3
162
163   setup:
164     image: juplo/toolbox
165     command:
166       - bash
167       - -c
168       - |
169         cub kafka-ready -b kafka-1:9092,kafka-2:9092,kafka-3:9092 3 60 > /dev/null 2>&1 || exit 1
170         if [ -e INITIALIZED ]
171         then
172           echo -n Bereits konfiguriert: 
173           cat INITIALIZED
174           kafka-topics --bootstrap-server kafka:9092 --describe --topic test
175         else
176           kafka-topics --bootstrap-server kafka:9092 \
177                        --delete \
178                        --if-exists \
179                        --topic test
180           kafka-topics --bootstrap-server kafka:9092 \
181                        --create \
182                        --topic test \
183                        --partitions 2 \
184                        --replication-factor 3 \
185                        --config min.insync.replicas=2 \
186           && echo Das Topic \'test\' wurde erfolgreich angelegt: \
187           && kafka-topics --bootstrap-server kafka:9092 --describe --topic test \
188           && date > INITIALIZED
189         fi
190     stop_grace_period: 0s
191     depends_on:
192       - cli
193
194   zoonavigator:
195     image: elkozmon/zoonavigator:1.1.2
196     ports:
197       - "8000:80"
198     environment:
199       HTTP_PORT: 80
200       CONNECTION_JUPLO_NAME: juplo
201       CONNECTION_JUPLO_CONN: zookeeper:2181
202       AUTO_CONNECT_CONNECTION_ID: JUPLO
203     depends_on:
204       - zookeeper
205
206   akhq:
207     image: tchiotludo/akhq:0.23.0
208     ports:
209       - 8888:8080
210     environment:
211       AKHQ_CONFIGURATION: |
212         akhq:
213           connections:
214             docker-kafka-server:
215               properties:
216                 bootstrap.servers: "kafka:9092"
217               schema-registry:
218                 url: "http://schema-registry:8085"
219               connect:
220                 - name: "connect"
221                   url: "http://connect:8083"
222     depends_on:
223       - kafka-1
224       - kafka-2
225       - kafka-3
226
227 volumes:
228   zookeeper-data:
229   zookeeper-log:
230   kafka-1-data:
231   kafka-2-data:
232   kafka-3-data: