From 2d663b1053908c717fdbd983fe143a24acac21a0 Mon Sep 17 00:00:00 2001
From: Kai Moritz <kai@juplo.de>
Date: Tue, 23 Apr 2024 08:55:35 +0200
Subject: [PATCH] =?utf8?q?Confluent=20S3=20Sink=20Connector=20hinzugef?=
 =?utf8?q?=C3=BCgt?=
MIME-Version: 1.0
Content-Type: text/plain; charset=utf8
Content-Transfer-Encoding: 8bit

* Der ausgepackte Connector selbst wurde absichtlich nicht mit in das
  Repository aufgenommen.
* Zur Vorführung/Überprüfung einen Befehl ergänzt, der die Beisiel-Konfig
  überprüft.
* Der Service `cli` hängt jetzt von dem Service `connect` ab, damit
  Kafka Connect in diesem Setup automatisch gestartet wird.
---
 .gitignore                           |  1 +
 README.sh                            | 17 ++---------------
 docker/confluentinc-kafka-connect-s3 |  1 +
 docker/docker-compose.yml            |  8 ++++----
 s3-sink-connector.json               | 17 +++++++++++++++++
 5 files changed, 25 insertions(+), 19 deletions(-)
 create mode 100644 .gitignore
 create mode 120000 docker/confluentinc-kafka-connect-s3
 create mode 100644 s3-sink-connector.json

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..6268bca
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+docker/confluentinc-kafka-connect-s3-*
diff --git a/README.sh b/README.sh
index 52f7101..98a581d 100755
--- a/README.sh
+++ b/README.sh
@@ -17,19 +17,6 @@ echo
 docker compose -f docker/docker-compose.yml exec minio mc mb /data/juplo
 
 echo
-echo "Hilfe-Ausgabe von kafkacat"
+echo "Validität der Konfiguration des S3-Sink-Connector prüfen"
 echo
-docker compose -f docker/docker-compose.yml exec -T cli kafkacat -h
-echo
-echo "Nachrichten schreiben mit kafkacat"
-echo
-docker compose -f docker/docker-compose.yml exec -T cli kafkacat -P -b kafka:9092 -t test << EOF
-Hallo Welt!
-Nachricht #1
-Nachricht #2
-Nachricht #3
-EOF
-echo
-echo "Nachrichten lesen mit kafkacat"
-echo
-docker compose -f docker/docker-compose.yml exec cli kafkacat -C -b kafka:9092 -t test -o beginning -e
+cat s3-sink-connector.json | jq .config | http -v put :8083/connector-plugins/io.confluent.connect.s3.S3SinkConnector/config/validate
diff --git a/docker/confluentinc-kafka-connect-s3 b/docker/confluentinc-kafka-connect-s3
new file mode 120000
index 0000000..96204d2
--- /dev/null
+++ b/docker/confluentinc-kafka-connect-s3
@@ -0,0 +1 @@
+confluentinc-kafka-connect-s3-10.5.11/
\ No newline at end of file
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 3a0b43b..36e14c8 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -107,9 +107,11 @@ services:
       CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
       CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
       CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
-      CONNECT_PLUGIN_PATH: /usr/share/java/
+      CONNECT_PLUGIN_PATH: /usr/share/java/plugins/
     ports:
       - 8083:8083
+    volumes:
+      - ./confluentinc-kafka-connect-s3:/usr/share/java/plugins/confluentinc-kafka-connect-s3:ro
     depends_on:
       - schema-registry
 
@@ -118,9 +120,7 @@ services:
     command: sleep infinity
     stop_grace_period: 0s
     depends_on:
-      - kafka-1
-      - kafka-2
-      - kafka-3
+      - connect
 
   setup:
     image: juplo/toolbox
diff --git a/s3-sink-connector.json b/s3-sink-connector.json
new file mode 100644
index 0000000..6cb9838
--- /dev/null
+++ b/s3-sink-connector.json
@@ -0,0 +1,17 @@
+{
+  "name": "s3-sink",
+  "config": {
+    "name": "s3-sink",
+    "connector.class": "io.confluent.connect.s3.S3SinkConnector",
+    "topics": "s3_topic",
+    "s3.region": "us-west-2",
+    "s3.bucket.name": "confluent-kafka-connect-s3-testing",
+    "s3.part.size": "5242880",
+    "flush.size": "3",
+    "storage.class": "io.confluent.connect.s3.storage.S3Storage",
+    "format.class": "io.confluent.connect.s3.format.avro.AvroFormat",
+    "schema.generator.class": "io.confluent.connect.storage.hive.schema.DefaultSchemaGenerator",
+    "partitioner.class": "io.confluent.connect.storage.partitioner.DefaultPartitioner",
+    "schema.compatibility": "NONE"
+  }
+}
-- 
2.20.1