X-Git-Url: http://juplo.de/gitweb/?a=blobdiff_plain;ds=sidebyside;f=src%2Fmain%2Fjava%2Fde%2Fjuplo%2Fkafka%2Fchat%2Fbackend%2Fimplementation%2Fkafka%2FDataChannel.java;h=e1754a157f978c33f5a617e16e9ea688f60fd5d6;hb=0f13dc5e88722ca7c238258747041d9663251356;hp=99beb438191d43c3f426e0f9c05e8ca97e3ebda2;hpb=27a2baceb8e94d41f14a6cd2598fa65e0ac514b9;p=demos%2Fkafka%2Fchat

diff --git a/src/main/java/de/juplo/kafka/chat/backend/implementation/kafka/DataChannel.java b/src/main/java/de/juplo/kafka/chat/backend/implementation/kafka/DataChannel.java
index 99beb438..e1754a15 100644
--- a/src/main/java/de/juplo/kafka/chat/backend/implementation/kafka/DataChannel.java
+++ b/src/main/java/de/juplo/kafka/chat/backend/implementation/kafka/DataChannel.java
@@ -7,10 +7,7 @@ import de.juplo.kafka.chat.backend.implementation.kafka.messages.AbstractMessage
 import de.juplo.kafka.chat.backend.implementation.kafka.messages.data.EventChatMessageReceivedTo;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.clients.consumer.Consumer;
-import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
-import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.*;
 import org.apache.kafka.clients.producer.Producer;
 import org.apache.kafka.clients.producer.ProducerRecord;
 import org.apache.kafka.common.TopicPartition;
@@ -31,13 +28,14 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
   private final Consumer<String, AbstractMessageTo> consumer;
   private final ZoneId zoneId;
   private final int numShards;
+  private final Duration pollingInterval;
   private final int bufferSize;
   private final Clock clock;
   private final boolean[] isShardOwned;
   private final long[] currentOffset;
   private final long[] nextOffset;
   private final Map<UUID, ChatRoomData>[] chatRoomData;
-  private final InfoChannel infoChannel;
+  private final ChannelMediator channelMediator;
   private final ShardingPublisherStrategy shardingPublisherStrategy;
 
   private boolean running;
@@ -52,9 +50,10 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     Consumer<String, AbstractMessageTo> dataChannelConsumer,
     ZoneId zoneId,
     int numShards,
+    Duration pollingInterval,
     int bufferSize,
     Clock clock,
-    InfoChannel infoChannel,
+    ChannelMediator channelMediator,
     ShardingPublisherStrategy shardingPublisherStrategy)
   {
     log.debug(
@@ -68,6 +67,7 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     this.producer = producer;
     this.zoneId = zoneId;
     this.numShards = numShards;
+    this.pollingInterval = pollingInterval;
     this.bufferSize = bufferSize;
     this.clock = clock;
     this.isShardOwned = new boolean[numShards];
@@ -77,7 +77,7 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     IntStream
         .range(0, numShards)
         .forEach(shard -> this.chatRoomData[shard] = new HashMap<>());
-    this.infoChannel = infoChannel;
+    this.channelMediator = channelMediator;
     this.shardingPublisherStrategy = shardingPublisherStrategy;
   }
 
@@ -144,7 +144,7 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
           currentOffset);
 
       consumer.seek(topicPartition, nextOffset[partition]);
-      infoChannel.sendShardAssignedEvent(partition);
+      channelMediator.shardAssigned(partition);
       shardingPublisherStrategy
           .publishOwnership(partition)
           .doOnSuccess(instanceId -> log.info(
@@ -155,7 +155,8 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
               "Could not publish instance {} as owner of shard {}: {}",
               instanceId,
               partition,
-              throwable))
+              throwable.toString()))
+          .onErrorComplete()
           .block();
     });
 
@@ -169,8 +170,9 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     {
       int partition = topicPartition.partition();
       isShardOwned[partition] = false;
+      nextOffset[partition] = consumer.position(topicPartition);
       log.info("Partition revoked: {} - next={}", partition, nextOffset[partition]);
-      infoChannel.sendShardRevokedEvent(partition);
+      channelMediator.shardRevoked(partition);
     });
   }
 
@@ -191,7 +193,7 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     {
       try
       {
-        ConsumerRecords<String, AbstractMessageTo> records = consumer.poll(Duration.ofMinutes(1));
+        ConsumerRecords<String, AbstractMessageTo> records = consumer.poll(pollingInterval);
         log.info("Fetched {} messages", records.count());
 
         if (loadInProgress)
@@ -265,12 +267,15 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     Message.MessageKey key = Message.MessageKey.of(chatMessageTo.getUser(), chatMessageTo.getId());
     Message message = new Message(key, offset, timestamp, chatMessageTo.getText());
 
-    ChatRoomData chatRoomData = this
-        .chatRoomData[partition]
-        .computeIfAbsent(chatRoomId, this::computeChatRoomData);
+    ChatRoomData chatRoomData = computeChatRoomData(chatRoomId, partition);
     KafkaChatMessageService kafkaChatRoomService =
         (KafkaChatMessageService) chatRoomData.getChatRoomService();
 
+    log.debug(
+        "Loaded message from partition={} at offset={}: {}",
+        partition,
+        offset,
+        message);
     kafkaChatRoomService.persistMessage(message);
   }
 
@@ -279,7 +284,12 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
     return IntStream
         .range(0, numShards)
         .filter(shard -> isShardOwned[shard])
-        .allMatch(shard -> nextOffset[shard] >= currentOffset[shard]);
+        .allMatch(shard ->
+        {
+          TopicPartition partition = new TopicPartition(topic, shard);
+          long position = consumer.position(partition);
+          return position >= currentOffset[shard];
+        });
   }
 
   private void pauseAllOwnedPartions()
@@ -300,6 +310,11 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
         .toArray();
   }
 
+  void createChatRoomData(ChatRoomInfo chatRoomInfo)
+  {
+    computeChatRoomData(chatRoomInfo.getId(), chatRoomInfo.getShard());
+  }
+
   Mono<ChatRoomData> getChatRoomData(int shard, UUID id)
   {
     if (loadInProgress)
@@ -312,16 +327,32 @@ public class DataChannel implements Runnable, ConsumerRebalanceListener
       return Mono.error(new ShardNotOwnedException(instanceId, shard));
     }
 
-    return infoChannel
-        .getChatRoomInfo(id)
-        .map(chatRoomInfo ->
-            chatRoomData[shard].computeIfAbsent(id, this::computeChatRoomData));
+    return Mono.justOrEmpty(chatRoomData[shard].get(id));
+  }
+
+  private ChatRoomData computeChatRoomData(UUID chatRoomId, int shard)
+  {
+    ChatRoomData chatRoomData = this.chatRoomData[shard].get(chatRoomId);
+
+    if (chatRoomData != null)
+    {
+      log.info(
+          "Ignoring request to create already existing ChatRoomData for {}",
+          chatRoomId);
+    }
+    else
+    {
+      log.info("Creating ChatRoomData {} with buffer-size {}", chatRoomId, bufferSize);
+      KafkaChatMessageService service = new KafkaChatMessageService(this, chatRoomId);
+      chatRoomData = new ChatRoomData(clock, service, bufferSize);
+      this.chatRoomData[shard].put(chatRoomId, chatRoomData);
+    }
+
+    return chatRoomData;
   }
 
-  private ChatRoomData computeChatRoomData(UUID chatRoomId)
+  ConsumerGroupMetadata getConsumerGroupMetadata()
   {
-    log.info("Creating ChatRoom {} with buffer-size {}", chatRoomId, bufferSize);
-    KafkaChatMessageService service = new KafkaChatMessageService(this, chatRoomId);
-    return new ChatRoomData(clock, service, bufferSize);
+    return consumer.groupMetadata();
   }
 }