From: Kai Moritz Date: Mon, 8 Apr 2024 14:57:38 +0000 (+0200) Subject: WIP X-Git-Url: http://juplo.de/gitweb/?a=commitdiff_plain;h=da1c9be9f3f13f646c34b34a37ef4b407e94ebc2;p=demos%2Fkafka%2Ftraining WIP --- diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c269dae..44c48b2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,7 +11,7 @@ services: - zookeeper-log:/var/lib/zookeeper/log kafka-1: - image: juplo/kafka + image: confluentinc/cp-kafka:7.5.1 environment: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9081 @@ -25,8 +25,6 @@ services: KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false" KAFKA_SUPER_USERS: User:ANONYMOUS KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO - KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter - KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091 KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: | org.apache.kafka.common.security.plain.PlainLoginModule required \ @@ -48,7 +46,7 @@ services: - zookeeper kafka-2: - image: juplo/kafka + image: confluentinc/cp-kafka:7.5.1 environment: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9082 @@ -62,8 +60,6 @@ services: KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false" KAFKA_SUPER_USERS: User:ANONYMOUS KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO - KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter - KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091 KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: | org.apache.kafka.common.security.plain.PlainLoginModule required \ @@ -90,7 +86,7 @@ services: - zookeeper kafka-3: - image: juplo/kafka + image: confluentinc/cp-kafka:7.5.1 environment: KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9083 @@ -104,8 +100,6 @@ services: KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false" KAFKA_SUPER_USERS: User:ANONYMOUS KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO - KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter - KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091 KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: | org.apache.kafka.common.security.plain.PlainLoginModule required \ @@ -126,51 +120,6 @@ services: depends_on: - zookeeper - kafka-4: - image: juplo/kafka - environment: - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - KAFKA_LISTENERS: BROKER://:9091, DOCKER://:9092, LOCALHOST://:9084 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: BROKER:PLAINTEXT, DOCKER:SASL_PLAINTEXT, LOCALHOST:SASL_PLAINTEXT - KAFKA_ADVERTISED_LISTENERS: BROKER://kafka-4:9091, DOCKER://kafka-4:9092, LOCALHOST://localhost:9084 - KAFKA_BROKER_ID: 4 - KAFKA_INTER_BROKER_LISTENER_NAME: BROKER - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" - KAFKA_AUTHORIZER_CLASS_NAME: kafka.security.authorizer.AclAuthorizer - KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: "false" - KAFKA_SUPER_USERS: User:ANONYMOUS - KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=INFO - KAFKA_METRIC_REPORTERS: com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter - KAFKA_CRUISE_CONTROL_METRICS_REPORTER_BOOTSTRAP_SERVERS: localhost:9091 - KAFKA_LISTENER_NAME_DOCKER_SASL_ENABLED_MECHANISMS: PLAIN - KAFKA_LISTENER_NAME_DOCKER_PLAIN_SASL_JAAS_CONFIG: | - org.apache.kafka.common.security.plain.PlainLoginModule required \ - user_schemaregistry="schemaregistry-secret" \ - user_connect="connect-secret" \ - user_ui="ui-secret" \ - user_client="client-secret"; - KAFKA_LISTENER_NAME_LOCALHOST_SASL_ENABLED_MECHANISMS: PLAIN - KAFKA_LISTENER_NAME_LOCALHOST_PLAIN_SASL_JAAS_CONFIG: | - org.apache.kafka.common.security.plain.PlainLoginModule required \ - user_client="client-secret"; - KAFKA_SASL_ENABLED_MECHANISMS: PLAIN, SCRAM-SHA-256, SCRAM-SHA-512 - volumes: - - kafka-4-data:/var/lib/kafka/data - ports: - - 9084:9084 - stop_grace_period: 120s - depends_on: - - zookeeper - - cruise-control: - image: juplo/cruise-control - environment: - BOOTSTRAP_SERVERS: kafka-1:9091,kafka-2:9091,kafka-3:9091 - ZOOKEEPER_CONNECT: zookeeper:2181/ - ports: - - "9090:9090" - schema-registry: image: confluentinc/cp-schema-registry:7.5.1 environment: @@ -264,7 +213,6 @@ services: echo -n Bereits konfiguriert: cat INITIALIZED kafka-topics --bootstrap-server kafka:9091 --describe --topic test - kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics else kafka-topics --bootstrap-server kafka:9091 \ --delete \ @@ -278,18 +226,6 @@ services: --config min.insync.replicas=2 \ && echo Das Topic \'test\' wurde erfolgreich angelegt: \ && kafka-topics --bootstrap-server kafka:9091 --describe --topic test - kafka-topics --bootstrap-server kafka:9091 \ - --delete \ - --if-exists \ - --topic __CruiseControlMetrics - kafka-topics --bootstrap-server kafka:9091 \ - --create \ - --topic __CruiseControlMetrics \ - --partitions 2 \ - --replication-factor 3 \ - --config min.insync.replicas=2 \ - && echo Das Topic \'__CruiseControlMetrics\' wurde erfolgreich angelegt: \ - && kafka-topics --bootstrap-server kafka:9091 --describe --topic __CruiseControlMetrics fi \ && date > INITIALIZED stop_grace_period: 0s @@ -332,53 +268,9 @@ services: - kafka-2 - kafka-3 - producer: - image: juplo/endless-stream-spring-producer:1.0-SNAPSHOT - environment: - producer.throttle-ms: 1 - spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092 - spring.kafka.client-id: producer - spring.kafka.template.default-topic: test - spring.kafka.producer.properties.linger.ms: 100 - spring.kafka.jaas.enabled: "true" - spring.kafka.properties.security.protocol: SASL_PLAINTEXT - spring.kafka.properties.sasl.mechanism: PLAIN - spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret"; - - consumer-1: - image: juplo/spring-consumer:1.0-SNAPSHOT - environment: - spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092 - spring.kafka.client-id: consumer-1 - spring.kafka.jaas.enabled: "true" - spring.kafka.properties.security.protocol: SASL_PLAINTEXT - spring.kafka.properties.sasl.mechanism: PLAIN - spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret"; - - consumer-2: - image: juplo/spring-consumer:1.0-SNAPSHOT - environment: - spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092 - spring.kafka.client-id: consumer-2 - spring.kafka.jaas.enabled: "true" - spring.kafka.properties.security.protocol: SASL_PLAINTEXT - spring.kafka.properties.sasl.mechanism: PLAIN - spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret"; - - consumer-3: - image: juplo/spring-consumer:1.0-SNAPSHOT - environment: - spring.kafka.bootstrap-servers: kafka-1:9092, kafka-2:9092, kafka-3:9092 - spring.kafka.client-id: consumer-3 - spring.kafka.jaas.enabled: "true" - spring.kafka.properties.security.protocol: SASL_PLAINTEXT - spring.kafka.properties.sasl.mechanism: PLAIN - spring.kafka.properties.sasl.jaas.config: org.apache.kafka.common.security.plain.PlainLoginModule required username="client" password="client-secret"; - volumes: zookeeper-data: zookeeper-log: kafka-1-data: kafka-2-data: kafka-3-data: - kafka-4-data: