Wenn kein gespeicherter Offset vorliegt, auto.offset.reset von Kafka nutzen

[demos/kafka/training] / src / main / java / de / juplo / kafka / EndlessConsumer.java

package de.juplo.kafka;

import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.RecordDeserializationException;
import org.apache.kafka.common.errors.WakeupException;

import javax.annotation.PreDestroy;
import java.time.Duration;
import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;


@Slf4j
@RequiredArgsConstructor
public class EndlessConsumer<K, V> implements ConsumerRebalanceListener, Runnable
{
  private final ExecutorService executor;
  private final PartitionStatisticsRepository repository;
  private final String id;
  private final String topic;
  private final Consumer<K, V> consumer;
  private final java.util.function.Consumer<ConsumerRecord<K, V>> handler;

  private final Lock lock = new ReentrantLock();
  private final Condition condition = lock.newCondition();
  private boolean running = false;
  private Exception exception;
  private long consumed = 0;

  private final Map<Integer, Map<String, Long>> seen = new HashMap<>();


  @Override
  public void onPartitionsRevoked(Collection<TopicPartition> partitions)
  {
    partitions.forEach(tp ->
    {
      Integer partition = tp.partition();
      Long newOffset = consumer.position(tp);
      log.info(
          "{} - removing partition: {}, offset of next message {})",
          id,
          partition,
          newOffset);
      Map<String, Long> removed = seen.remove(partition);
      for (String key : removed.keySet())
      {
        log.info(
            "{} - Seen {} messages for partition={}|key={}",
            id,
            removed.get(key),
            partition,
            key);
      }
      repository.save(new StatisticsDocument(partition, removed, consumer.position(tp)));
    });
  }

  @Override
  public void onPartitionsAssigned(Collection<TopicPartition> partitions)
  {
    partitions.forEach(tp ->
    {
      Integer partition = tp.partition();
      Long offset = consumer.position(tp);
      log.info("{} - adding partition: {}, offset={}", id, partition, offset);
      StatisticsDocument document =
          repository
              .findById(Integer.toString(partition))
              .orElse(new StatisticsDocument(partition));
      if (document.offset >= 0)
      {
        // Only seek, if a stored offset was found
        // Otherwise: Use initial offset, generated by Kafka
        consumer.seek(tp, document.offset);
      }
      seen.put(partition, document.statistics);
    });
  }


  @Override
  public void run()
  {
    try
    {
      log.info("{} - Subscribing to topic {}", id, topic);
      consumer.subscribe(Arrays.asList(topic), this);

      while (true)
      {
        ConsumerRecords<K, V> records =
            consumer.poll(Duration.ofSeconds(1));

        // Do something with the data...
        log.info("{} - Received {} messages", id, records.count());
        for (ConsumerRecord<K, V> record : records)
        {
          log.info(
              "{} - {}: {}/{} - {}={}",
              id,
              record.offset(),
              record.topic(),
              record.partition(),
              record.key(),
              record.value()
          );

          handler.accept(record);

          consumed++;

          Integer partition = record.partition();
          String key = record.key() == null ? "NULL" : record.key().toString();
          Map<String, Long> byKey = seen.get(partition);

          if (!byKey.containsKey(key))
            byKey.put(key, 0l);

          long seenByKey = byKey.get(key);
          seenByKey++;
          byKey.put(key, seenByKey);
        }

        seen.forEach((partiton, statistics) -> repository.save(
            new StatisticsDocument(
                partiton,
                statistics,
                consumer.position(new TopicPartition(topic, partiton)))));
      }
    }
    catch(WakeupException e)
    {
      log.info("{} - RIIING! Request to stop consumption - commiting current offsets!", id);
      shutdown();
    }
    catch(RecordDeserializationException e)
    {
      TopicPartition tp = e.topicPartition();
      long offset = e.offset();
      log.error(
          "{} - Could not deserialize  message on topic {} with offset={}: {}",
          id,
          tp,
          offset,
          e.getCause().toString());

      shutdown(e);
    }
    catch(Exception e)
    {
      log.error("{} - Unexpected error: {}", id, e.toString(), e);
      shutdown(e);
    }
    finally
    {
      log.info("{} - Consumer-Thread exiting", id);
    }
  }

  private void shutdown()
  {
    shutdown(null);
  }

  private void shutdown(Exception e)
  {
    lock.lock();
    try
    {
      try
      {
        log.info("{} - Unsubscribing from topic {}", id, topic);
        consumer.unsubscribe();
      }
      catch (Exception ue)
      {
        log.error(
            "{} - Error while unsubscribing from topic {}: {}",
            id,
            topic,
            ue.toString());
      }
      finally
      {
        running = false;
        exception = e;
        condition.signal();
      }
    }
    finally
    {
      lock.unlock();
    }
  }

  public Map<Integer, Map<String, Long>> getSeen()
  {
    return seen;
  }

  public void start()
  {
    lock.lock();
    try
    {
      if (running)
        throw new IllegalStateException("Consumer instance " + id + " is already running!");

      log.info("{} - Starting - consumed {} messages before", id, consumed);
      running = true;
      exception = null;
      executor.submit(this);
    }
    finally
    {
      lock.unlock();
    }
  }

  public synchronized void stop() throws ExecutionException, InterruptedException
  {
    lock.lock();
    try
    {
      if (!running)
        throw new IllegalStateException("Consumer instance " + id + " is not running!");

      log.info("{} - Stopping", id);
      consumer.wakeup();
      condition.await();
      log.info("{} - Stopped - consumed {} messages so far", id, consumed);
    }
    finally
    {
      lock.unlock();
    }
  }

  @PreDestroy
  public void destroy() throws ExecutionException, InterruptedException
  {
    log.info("{} - Destroy!", id);
    log.info("{}: Consumed {} messages in total, exiting!", id, consumed);
  }

  public boolean running()
  {
    lock.lock();
    try
    {
      return running;
    }
    finally
    {
      lock.unlock();
    }
  }

  public Optional<Exception> exitStatus()
  {
    lock.lock();
    try
    {
      if (running)
        throw new IllegalStateException("No exit-status available: Consumer instance " + id + " is running!");

      return Optional.ofNullable(exception);
    }
    finally
    {
      lock.unlock();
    }
  }
}