From: Kai Moritz <kai@juplo.de>
Date: Tue, 23 Apr 2024 06:55:35 +0000 (+0200)
Subject: Confluent S3 Sink Connector hinzugefügt
X-Git-Url: http://juplo.de/gitweb/?a=commitdiff_plain;h=2d663b1053908c717fdbd983fe143a24acac21a0;p=demos%2Fkafka%2Ftraining

Confluent S3 Sink Connector hinzugefügt

* Der ausgepackte Connector selbst wurde absichtlich nicht mit in das
  Repository aufgenommen.
* Zur Vorführung/Überprüfung einen Befehl ergänzt, der die Beisiel-Konfig
  überprüft.
* Der Service `cli` hängt jetzt von dem Service `connect` ab, damit
  Kafka Connect in diesem Setup automatisch gestartet wird.
---

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6268bca
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+docker/confluentinc-kafka-connect-s3-*
diff --git a/README.sh b/README.sh
index 52f7101..98a581d 100755
--- a/README.sh
+++ b/README.sh
@@ -17,19 +17,6 @@ echo
 docker compose -f docker/docker-compose.yml exec minio mc mb /data/juplo
 
 echo
-echo "Hilfe-Ausgabe von kafkacat"
+echo "Validität der Konfiguration des S3-Sink-Connector prüfen"
 echo
-docker compose -f docker/docker-compose.yml exec -T cli kafkacat -h
-echo
-echo "Nachrichten schreiben mit kafkacat"
-echo
-docker compose -f docker/docker-compose.yml exec -T cli kafkacat -P -b kafka:9092 -t test << EOF
-Hallo Welt!
-Nachricht #1
-Nachricht #2
-Nachricht #3
-EOF
-echo
-echo "Nachrichten lesen mit kafkacat"
-echo
-docker compose -f docker/docker-compose.yml exec cli kafkacat -C -b kafka:9092 -t test -o beginning -e
+cat s3-sink-connector.json | jq .config | http -v put :8083/connector-plugins/io.confluent.connect.s3.S3SinkConnector/config/validate
diff --git a/docker/confluentinc-kafka-connect-s3 b/docker/confluentinc-kafka-connect-s3
new file mode 120000
index 0000000..96204d2
--- /dev/null
+++ b/docker/confluentinc-kafka-connect-s3
@@ -0,0 +1 @@
+confluentinc-kafka-connect-s3-10.5.11/
\ No newline at end of file
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 3a0b43b..36e14c8 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -107,9 +107,11 @@ services:
       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_PLUGIN_PATH: /usr/share/java/
+      CONNECT_PLUGIN_PATH: /usr/share/java/plugins/
     ports:
       - 8083:8083
+    volumes:
+      - ./confluentinc-kafka-connect-s3:/usr/share/java/plugins/confluentinc-kafka-connect-s3:ro
     depends_on:
       - schema-registry
 
@@ -118,9 +120,7 @@ services:
     command: sleep infinity
     stop_grace_period: 0s
     depends_on:
-      - kafka-1
-      - kafka-2
-      - kafka-3
+      - connect
 
   setup:
     image: juplo/toolbox
diff --git a/s3-sink-connector.json b/s3-sink-connector.json
new file mode 100644
index 0000000..6cb9838
--- /dev/null
+++ b/s3-sink-connector.json
@@ -0,0 +1,17 @@
+{
+  "name": "s3-sink",
+  "config": {
+    "name": "s3-sink",
+    "connector.class": "io.confluent.connect.s3.S3SinkConnector",
+    "topics": "s3_topic",
+    "s3.region": "us-west-2",
+    "s3.bucket.name": "confluent-kafka-connect-s3-testing",
+    "s3.part.size": "5242880",
+    "flush.size": "3",
+    "storage.class": "io.confluent.connect.s3.storage.S3Storage",
+    "format.class": "io.confluent.connect.s3.format.avro.AvroFormat",
+    "schema.generator.class": "io.confluent.connect.storage.hive.schema.DefaultSchemaGenerator",
+    "partitioner.class": "io.confluent.connect.storage.partitioner.DefaultPartitioner",
+    "schema.compatibility": "NONE"
+  }
+}