</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
</plugins>
</build>
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
-import org.apache.kafka.clients.consumer.*;
+import org.apache.kafka.clients.consumer.Consumer;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.errors.RecordDeserializationException;
import org.apache.kafka.common.errors.WakeupException;
import org.springframework.boot.test.web.server.LocalServerPort;
import org.springframework.kafka.test.context.EmbeddedKafka;
-import static de.juplo.kafka.ApplicationTests.TOPIC;
+import static de.juplo.kafka.ApplicationIT.TOPIC;
@SpringBootTest(
import org.springframework.beans.factory.annotation.Autowired;
import java.util.*;
+
import java.util.function.Consumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
public ApplicationTests()
{
super(new ApplicationTestRecrodGenerator());
- ((ApplicationTestRecrodGenerator)recordGenerator).tests = this;
+ ((ApplicationTestRecrodGenerator) recordGenerator).tests = this;
}
final int[] numbers = {1, 77, 33, 2, 66, 666, 11};
final String[] dieWilden13 =
- IntStream
- .range(1, 14)
- .mapToObj(i -> "seeräuber-" + i)
- .toArray(i -> new String[i]);
+ IntStream
+ .range(1, 14)
+ .mapToObj(i -> "seeräuber-" + i)
+ .toArray(i -> new String[i]);
final StringSerializer stringSerializer = new StringSerializer();
final Bytes calculateMessage = new Bytes(stringSerializer.serialize(TOPIC, "CALCULATE"));
@Override
public int generate(
- boolean poisonPills,
- boolean logicErrors,
-
Consumer<ProducerRecord<Bytes, Bytes>> messageSender)
+ boolean poisonPills,
+ boolean logicErrors,
+ Consumer<ProducerRecord<Bytes, Bytes>> messageSender)
{
counter = 0;
state =
- Arrays
- .stream(dieWilden13)
- .collect(Collectors.toMap(
- seeräuber -> seeräuber,
- seeräuber -> new LinkedList()));
-
- int number[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
- int message[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 };
+ Arrays
+ .stream(dieWilden13)
+ .collect(Collectors.toMap(
+ seeräuber -> seeräuber,
+ seeräuber -> new LinkedList()));
+
+ int number[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+ int message[] = {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
int next = 0;
for (int pass = 0; pass < 333; pass++)
{
- for (int i = 0; i<13; i++)
+ for (int i = 0; i < 13; i++)
{
String seeräuber = dieWilden13[i];
Bytes key = new Bytes(stringSerializer.serialize(TOPIC, seeräuber));
send(key, calculateMessage, fail(logicErrors, pass, counter), messageSender);
state.get(seeräuber).add(new AdderResult(number[i], (number[i] + 1) * number[i] / 2));
// Pick next number to calculate
- number[i] = numbers[next++%numbers.length];
+ number[i] = numbers[next++ % numbers.length];
message[i] = 1;
log.debug("Seeräuber {} will die Summe für {} berechnen", seeräuber, number[i]);
}
return counter;
}
- boolean fail (boolean logicErrors, int pass, int counter)
+ boolean fail(boolean logicErrors, int pass, int counter)
{
- return logicErrors && pass > 300 && counter%77 == 0;
+ return logicErrors && pass > 300 && counter % 77 == 0;
}
void send(
- Bytes key,
- Bytes value,
- boolean fail,
-
Consumer<ProducerRecord<Bytes, Bytes>> messageSender)
+ Bytes key,
+ Bytes value,
+ boolean fail,
+ Consumer<ProducerRecord<Bytes, Bytes>> messageSender)
{
counter++;
@Override
public void assertBusinessLogic()
{
- for (int i=0; i<PARTITIONS; i++)
+ for (int i = 0; i < PARTITIONS; i++)
{
StateDocument stateDocument =
- tests.stateRepository.findById(Integer.toString(i)).get();
+ tests.stateRepository.findById(Integer.toString(i)).get();
stateDocument
- .results
- .entrySet()
- .stream()
- .forEach(entry ->
- {
- String user = entry.getKey();
-
List<AdderResult> resultsForUser = entry.getValue();
+ .results
+ .entrySet()
+ .stream()
+ .forEach(entry ->
+ {
+ String user = entry.getKey();
+ List<AdderResult> resultsForUser = entry.getValue();
- for (int j=0; j < resultsForUser.size(); j++)
+ for (int j = 0; j < resultsForUser.size(); j++)
+ {
+ if (!(j < state.get(user).size()))
{
- if (!(j < state.get(user).size()))
- {
- break;
- }
-
- assertThat(resultsForUser.get(j))
- .as("Unexpected results calculation %d of user %s", j, user)
- .isEqualTo(state.get(user).get(j));
+ break;
}
- assertThat(state.get(user))
- .as("More results calculated for user %s as expected", user)
- .containsAll(resultsForUser);
- });
+ assertThat(resultsForUser.get(j))
+ .as("Unexpected results calculation %d of user %s", j, user)
+ .isEqualTo(state.get(user).get(j));
+ }
+
+ assertThat(state.get(user))
+ .as("More results calculated for user %s as expected", user)
+ .containsAll(resultsForUser);
+ });
}
}
}
-}
+}
\ No newline at end of file
import org.springframework.boot.test.autoconfigure.data.mongo.AutoConfigureDataMongo;
import org.springframework.boot.test.context.ConfigDataApplicationContextInitializer;
import org.springframework.boot.test.context.TestConfiguration;
+import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Import;
import org.springframework.kafka.test.context.EmbeddedKafka;
import org.springframework.test.context.TestPropertySource;
import java.time.Duration;
import java.util.*;
-import java.util.concurrent.ExecutorService;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.stream.Collectors;
@Autowired
- KafkaConsumer<K, V> kafkaConsumer;
+ org.apache.kafka.clients.consumer.Consumer<K, V> kafkaConsumer;
@Autowired
Consumer<ConsumerRecord<K, V>> consumer;
@Autowired
- ApplicationProperties properties;
- @Autowired
- ExecutorService executor;
+ ApplicationProperties applicationProperties;
@Autowired
MongoClient mongoClient;
@Autowired
@Autowired
RebalanceListener rebalanceListener;
@Autowired
- RecordHandler<K, V> recordHandler;
+ TestRecordHandler<K, V> recordHandler;
+ @Autowired
+ EndlessConsumer<K, V> endlessConsumer;
KafkaProducer<Bytes, Bytes> testRecordProducer;
KafkaConsumer<Bytes, Bytes> offsetConsumer;
- EndlessConsumer<K, V> endlessConsumer;
Map<TopicPartition, Long> oldOffsets;
- Map<TopicPartition, Long> seenOffsets;
- Set<ConsumerRecord<K, V>> receivedRecords;
final RecordGenerator recordGenerator;
await(numberOfGeneratedMessages + " records received")
.atMost(Duration.ofSeconds(30))
.pollInterval(Duration.ofSeconds(1))
- .until(() -> receivedRecords.size() >= numberOfGeneratedMessages);
+ .until(() -> recordHandler.receivedRecords.size() >= numberOfGeneratedMessages);
await("Offsets committed")
.atMost(Duration.ofSeconds(10))
.untilAsserted(() ->
{
checkSeenOffsetsForProgress();
- assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
+ assertSeenOffsetsEqualCommittedOffsets(recordHandler.seenOffsets);
});
assertThatExceptionOfType(IllegalStateException.class)
.until(() -> !endlessConsumer.running());
checkSeenOffsetsForProgress();
- assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
+ assertSeenOffsetsEqualCommittedOffsets(recordHandler.seenOffsets);
endlessConsumer.start();
await("Consumer failed")
.until(() -> !endlessConsumer.running());
checkSeenOffsetsForProgress();
- assertSeenOffsetsEqualCommittedOffsets(seenOffsets);
- assertThat(receivedRecords.size())
+ assertSeenOffsetsEqualCommittedOffsets(recordHandler.seenOffsets);
+ assertThat(recordHandler.receivedRecords.size())
.describedAs("Received not all sent events")
.isLessThan(numberOfGeneratedMessages);
.until(() -> !endlessConsumer.running());
checkSeenOffsetsForProgress();
- assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
+ assertSeenOffsetsAreBehindCommittedOffsets(recordHandler.seenOffsets);
endlessConsumer.start();
await("Consumer failed")
.pollInterval(Duration.ofSeconds(1))
.until(() -> !endlessConsumer.running());
- assertSeenOffsetsAreBehindCommittedOffsets(seenOffsets);
+ assertSeenOffsetsAreBehindCommittedOffsets(recordHandler.seenOffsets);
assertThatNoException()
.describedAs("Consumer should not be running")
partitions().forEach(tp ->
{
Long oldOffset = oldOffsets.get(tp) + 1;
- Long newOffset = seenOffsets.get(tp) + 1;
+ Long newOffset = recordHandler.seenOffsets.get(tp) + 1;
if (!oldOffset.equals(newOffset))
{
log.debug("Progress for {}: {} -> {}", tp, oldOffset, newOffset);
{
Properties props;
props = new Properties();
- props.put("bootstrap.servers", properties.getBootstrapServer());
+ props.put("bootstrap.servers", applicationProperties.getBootstrapServer());
props.put("linger.ms", 100);
props.put("key.serializer", BytesSerializer.class.getName());
props.put("value.serializer", BytesSerializer.class.getName());
testRecordProducer = new KafkaProducer<>(props);
props = new Properties();
- props.put("bootstrap.servers", properties.getBootstrapServer());
+ props.put("bootstrap.servers", applicationProperties.getBootstrapServer());
props.put("client.id", "OFFSET-CONSUMER");
- props.put("group.id", properties.getGroupId());
+ props.put("group.id", applicationProperties.getGroupId());
props.put("key.deserializer", BytesDeserializer.class.getName());
props.put("value.deserializer", BytesDeserializer.class.getName());
offsetConsumer = new KafkaConsumer<>(props);
seekToEnd();
oldOffsets = new HashMap<>();
- seenOffsets = new HashMap<>();
- receivedRecords = new HashSet<>();
+ recordHandler.seenOffsets = new HashMap<>();
+ recordHandler.receivedRecords = new HashSet<>();
doForCurrentOffsets((tp, offset) ->
{
oldOffsets.put(tp, offset - 1);
- seenOffsets.put(tp, offset - 1);
+ recordHandler.seenOffsets.put(tp, offset - 1);
});
- TestRecordHandler<K, V> captureOffsetAndExecuteTestHandler =
- new TestRecordHandler<K, V>(recordHandler)
- {
- @Override
- public void onNewRecord(ConsumerRecord<K, V> record)
- {
- seenOffsets.put(
- new TopicPartition(record.topic(), record.partition()),
- record.offset());
- receivedRecords.add(record);
- }
- };
-
- endlessConsumer =
- new EndlessConsumer<>(
- executor,
- properties.getClientId(),
- properties.getTopic(),
- kafkaConsumer,
- rebalanceListener,
- captureOffsetAndExecuteTestHandler);
-
endlessConsumer.start();
}
@AfterEach
public void deinit()
{
+ try
+ {
+ endlessConsumer.stop();
+ }
+ catch (Exception e)
+ {
+ log.debug("{}", e.toString());
+ }
+
try
{
testRecordProducer.close();
@Import(ApplicationConfiguration.class)
public static class Configuration
{
+ @Bean
+ public RecordHandler recordHandler(RecordHandler applicationRecordHandler)
+ {
+ return new TestRecordHandler(applicationRecordHandler);
+ }
}
}
import lombok.RequiredArgsConstructor;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
+
+import java.util.Map;
+import java.util.Set;
@RequiredArgsConstructor
-public abstract class TestRecordHandler<K, V> implements RecordHandler<K, V>
+public class TestRecordHandler<K, V> implements RecordHandler<K, V>
{
private final RecordHandler<K, V> handler;
+ Map<TopicPartition, Long> seenOffsets;
+ Set<ConsumerRecord<K, V>> receivedRecords;
- public abstract void onNewRecord(ConsumerRecord<K, V> record);
+ public void onNewRecord(ConsumerRecord<K, V> record)
+ {
+ seenOffsets.put(
+ new TopicPartition(record.topic(), record.partition()),
+ record.offset());
+ receivedRecords.add(record);
+ }
@Override
public void accept(ConsumerRecord<K, V> record)
projects / demos / kafka / training / commitdiff