import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.common.TopicPartition;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
+import java.util.concurrent.CountDownLatch;
@RequiredArgsConstructor
@Slf4j
-public class ApplicationRebalanceListener implements PollIntervalAwareConsumerRebalanceListener
+public class ApplicationRebalanceListener implements ConsumerRebalanceListener
{
+ private final Consumer consumer;
private final ApplicationRecordHandler recordHandler;
private final AdderResults adderResults;
private final StateRepository stateRepository;
private final String id;
- private final Clock clock;
- private final Duration commitInterval;
private final Set<Integer> partitions = new HashSet<>();
- private Instant lastCommit = Instant.EPOCH;
-
@Override
public void onPartitionsAssigned(Collection<TopicPartition> partitions)
{
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions)
{
+ log.info("{} - Commiting offsets for all previously assigned partitions", id);
+ CountDownLatch commitDone = new CountDownLatch(1);
+ consumer.commitAsync((offsets, e) ->
+ {
+ commitDone.countDown();
+ if (e == null)
+ {
+ log.error("{} - Could not commit offsets to Kafka!", id, e);
+ }
+ else
+ {
+ offsets.entrySet().stream().forEach(entry ->
+ {
+ log.info("{} - Commited offset for {}: {}", id, entry.getKey(), entry.getValue());
+ });
+ }
+ });
+
partitions.forEach(tp ->
{
Integer partition = tp.partition();
Map<String, List<AdderResult>> results = adderResults.removePartition(partition);
stateRepository.save(new StateDocument(partition, state, results));
});
- }
-
- @Override
- public void beforeNextPoll()
- {
- if (lastCommit.plus(commitInterval).isBefore(clock.instant()))
+ try
+ {
+ log.debug("{} - Waiting for async commit to complete", id);
+ commitDone.await();
+ }
+ catch (InterruptedException e)
{
- log.debug("Storing data, last commit: {}", lastCommit);
- partitions.forEach(partition -> stateRepository.save(
- new StateDocument(
- partition,
- recordHandler.getState(partition).getState(),
- adderResults.getState(partition))));
- lastCommit = clock.instant();
+ log.warn(
+ "{} - Interrupted while waiting for async commit in onPartitionsRevoked({})",
+ id,
+ partitions,
+ e);
}
}
}