From ab53ded9c9e2372d63198b666a1d9aa6ad027bd5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 6 Jan 2025 15:48:03 +0100 Subject: [PATCH 01/72] Simplify writing to output topics --- README.md | 6 +- .../test/java/com/bakdata/kafka/CliTest.java | 6 +- .../java/com/bakdata/kafka/CloseFlagApp.java | 7 +- .../kafka/test_applications/Mirror.java | 8 +- .../kafka/test_applications/WordCount.java | 12 +- .../com/bakdata/kafka/ConfiguredConsumed.java | 69 ++ .../com/bakdata/kafka/ConfiguredProduced.java | 66 ++ .../kafka/ImprovedCogroupedKStream.java | 68 ++ .../kafka/ImprovedCogroupedStreamImpl.java | 90 +++ .../bakdata/kafka/ImprovedKGroupedStream.java | 86 +++ .../kafka/ImprovedKGroupedStreamImpl.java | 125 ++++ .../bakdata/kafka/ImprovedKGroupedTable.java | 78 ++ .../kafka/ImprovedKGroupedTableImpl.java | 111 +++ .../com/bakdata/kafka/ImprovedKStream.java | 391 ++++++++++ .../bakdata/kafka/ImprovedKStreamImpl.java | 670 ++++++++++++++++++ .../com/bakdata/kafka/ImprovedKTable.java | 246 +++++++ .../com/bakdata/kafka/ImprovedKTableImpl.java | 381 ++++++++++ ...provedSessionWindowedCogroupedKStream.java | 52 ++ ...vedSessionWindowedCogroupedStreamImpl.java | 70 ++ .../kafka/ImprovedSessionWindowedKStream.java | 91 +++ .../ImprovedSessionWindowedStreamImpl.java | 123 ++++ .../ImprovedTimeWindowedCogroupedKStream.java | 50 ++ ...provedTimeWindowedCogroupedStreamImpl.java | 64 ++ .../kafka/ImprovedTimeWindowedKStream.java | 88 +++ .../kafka/ImprovedTimeWindowedStreamImpl.java | 120 ++++ .../com/bakdata/kafka/StreamsContext.java | 89 +++ .../com/bakdata/kafka/TopologyBuilder.java | 112 ++- .../kafka/ExecutableStreamsAppTest.java | 4 +- .../kafka/integration/StreamsRunnerTest.java | 8 +- .../ComplexTopologyApplication.java | 14 +- .../test_applications/LabeledInputTopics.java | 8 +- .../kafka/test_applications/Mirror.java | 8 +- .../test_applications/MirrorKeyWithAvro.java | 8 +- .../MirrorValueWithAvro.java | 8 +- .../MirrorWithNonDefaultSerde.java | 21 +- .../kafka/test_applications/WordCount.java | 12 +- .../test_applications/WordCountPattern.java | 12 +- 37 files changed, 3292 insertions(+), 90 deletions(-) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java diff --git a/README.md b/README.md index ab63f359..e8ae80ae 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ Create a subclass of `KafkaStreamsApplication` and implement the abstract method and `getUniqueAppId()`. You can define the topology of your application in `buildTopology()`. ```java +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.KafkaStreamsApplication; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -64,7 +65,6 @@ import com.bakdata.kafka.StreamsTopicConfig; import com.bakdata.kafka.TopologyBuilder; import java.util.Map; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; public class MyStreamsApplication extends KafkaStreamsApplication { public static void main(final String[] args) { @@ -76,11 +76,11 @@ public class MyStreamsApplication extends KafkaStreamsApplication { return new StreamsApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); + final ImprovedKStream input = builder.streamInput(); // your topology - input.to(builder.getTopics().getOutputTopic()); + input.toOutput(); } @Override diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CliTest.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CliTest.java index 05060156..2b7a66e1 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CliTest.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CliTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -187,7 +187,7 @@ public StreamsApp createApp() { return new StreamsApp() { @Override public void buildTopology(final TopologyBuilder builder) { - builder.streamInput().to(builder.getTopics().getOutputTopic()); + builder.streamInput().toOutputTopic(); } @Override @@ -256,7 +256,7 @@ void shouldExitWithSuccessCodeOnShutdown() throws InterruptedException { @Override public void buildTopology(final TopologyBuilder builder) { builder.streamInput(Consumed.with(Serdes.ByteArray(), Serdes.ByteArray())) - .to(builder.getTopics().getOutputTopic()); + .toOutputTopic(); } @Override diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java index cdd7b0d9..acf7aaee 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,6 @@ import lombok.NoArgsConstructor; import lombok.Setter; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; @NoArgsConstructor @Getter @@ -49,8 +48,8 @@ public StreamsApp createApp() { return new StreamsApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); - input.to(builder.getTopics().getOutputTopic()); + final ImprovedKStream input = builder.streamInput(); + input.toOutputTopic(); } @Override diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java index 54aee24c..8833fae6 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,20 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; import com.bakdata.kafka.TopologyBuilder; import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; @NoArgsConstructor public class Mirror implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); - input.to(builder.getTopics().getOutputTopic()); + final ImprovedKStream input = builder.streamInput(); + input.toOutputTopic(); } @Override diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java index 58f6d7af..3418860f 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,8 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.ImprovedKTable; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -33,8 +35,6 @@ import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Produced; @@ -43,15 +43,15 @@ public class WordCount implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream textLines = builder.streamInput(); + final ImprovedKStream textLines = builder.streamInput(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); - final KTable wordCounts = textLines + final ImprovedKTable wordCounts = textLines .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) .groupBy((key, word) -> word) .count(Materialized.as("counts")); - wordCounts.toStream().to(builder.getTopics().getOutputTopic(), Produced.valueSerde(Serdes.Long())); + wordCounts.toStream().toOutputTopic(Produced.valueSerde(Serdes.Long())); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java new file mode 100644 index 00000000..12d894b6 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java @@ -0,0 +1,69 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.processor.TimestampExtractor; + +@With +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class ConfiguredConsumed { + + private final Preconfigured> keySerde; + private final Preconfigured> valueSerde; + private final TimestampExtractor timestampExtractor; + private final AutoOffsetReset offsetResetPolicy; + private final String name; + + public static ConfiguredConsumed keySerde(final Preconfigured> keySerde) { + return with(keySerde, null); + } + + public static ConfiguredConsumed valueSerde(final Preconfigured> valueSerde) { + return with(null, valueSerde); + } + + public static ConfiguredConsumed with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ConfiguredConsumed<>(keySerde, valueSerde, null, null, null); + } + + public static ConfiguredConsumed as(final String processorName) { + return new ConfiguredConsumed<>(null, null, null, null, processorName); + } + + Consumed configure(final Configurator configurator) { + return Consumed.as(this.name) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) + .withOffsetResetPolicy(this.offsetResetPolicy) + .withTimestampExtractor(this.timestampExtractor); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java new file mode 100644 index 00000000..7d350a00 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java @@ -0,0 +1,66 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.processor.StreamPartitioner; + +@With +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class ConfiguredProduced { + + private final Preconfigured> keySerde; + private final Preconfigured> valueSerde; + private final StreamPartitioner partitioner; + private final String processorName; + + public static ConfiguredProduced keySerde(final Preconfigured> keySerde) { + return with(keySerde, null); + } + + public static ConfiguredProduced valueSerde(final Preconfigured> valueSerde) { + return with(null, valueSerde); + } + + public static ConfiguredProduced with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ConfiguredProduced<>(keySerde, valueSerde, null, null); + } + + public static ConfiguredProduced as(final String processorName) { + return new ConfiguredProduced<>(null, null, null, processorName); + } + + Produced configure(final Configurator configurator) { + return Produced.as(this.processorName) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) + .withStreamPartitioner(this.partitioner); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java new file mode 100644 index 00000000..89a4e36e --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -0,0 +1,68 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.CogroupedKStream; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.SlidingWindows; +import org.apache.kafka.streams.kstream.Window; +import org.apache.kafka.streams.kstream.Windows; +import org.apache.kafka.streams.state.KeyValueStore; + +public interface ImprovedCogroupedKStream extends CogroupedKStream { + + @Override + ImprovedCogroupedKStream cogroup(KGroupedStream groupedStream, + Aggregator aggregator); + + @Override + ImprovedKTable aggregate(Initializer initializer); + + @Override + ImprovedKTable aggregate(Initializer initializer, Named named); + + @Override + ImprovedKTable aggregate(Initializer initializer, + Materialized> materialized); + + @Override + ImprovedKTable aggregate(Initializer initializer, Named named, + Materialized> materialized); + + @Override + ImprovedTimeWindowedCogroupedKStream windowedBy(Windows windows); + + @Override + ImprovedTimeWindowedCogroupedKStream windowedBy(SlidingWindows windows); + + @Override + ImprovedSessionWindowedCogroupedKStream windowedBy(SessionWindows windows); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java new file mode 100644 index 00000000..ebaddeca --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java @@ -0,0 +1,90 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.CogroupedKStream; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.SlidingWindows; +import org.apache.kafka.streams.kstream.Window; +import org.apache.kafka.streams.kstream.Windows; +import org.apache.kafka.streams.state.KeyValueStore; + +@RequiredArgsConstructor +class ImprovedCogroupedStreamImpl implements ImprovedCogroupedKStream { + + private final @NonNull CogroupedKStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedCogroupedKStream cogroup(final KGroupedStream groupedStream, + final Aggregator aggregator) { + return this.context.newCogroupedStream(this.wrapped.cogroup(groupedStream, aggregator)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer) { + return this.context.newTable(this.wrapped.aggregate(initializer)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, final Named named) { + return this.context.newTable(this.wrapped.aggregate(initializer, named)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, materialized)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, named, materialized)); + } + + @Override + public ImprovedTimeWindowedCogroupedKStream windowedBy(final Windows windows) { + return this.context.newTimeWindowedCogroupedStream(this.wrapped.windowedBy(windows)); + } + + @Override + public ImprovedTimeWindowedCogroupedKStream windowedBy(final SlidingWindows windows) { + return this.context.newTimeWindowedCogroupedStream(this.wrapped.windowedBy(windows)); + } + + @Override + public ImprovedSessionWindowedCogroupedKStream windowedBy(final SessionWindows windows) { + return this.context.newSessionWindowedCogroupedStream(this.wrapped.windowedBy(windows)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java new file mode 100644 index 00000000..3fc76ed8 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -0,0 +1,86 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.SlidingWindows; +import org.apache.kafka.streams.kstream.Window; +import org.apache.kafka.streams.kstream.Windows; +import org.apache.kafka.streams.state.KeyValueStore; + +public interface ImprovedKGroupedStream extends KGroupedStream { + + @Override + ImprovedKTable count(); + + @Override + ImprovedKTable count(Named named); + + @Override + ImprovedKTable count(Materialized> materialized); + + @Override + ImprovedKTable count(Named named, Materialized> materialized); + + @Override + ImprovedKTable reduce(Reducer reducer); + + @Override + ImprovedKTable reduce(Reducer reducer, Materialized> materialized); + + @Override + ImprovedKTable reduce(Reducer reducer, Named named, + Materialized> materialized); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + Materialized> materialized); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + Named named, Materialized> materialized); + + @Override + ImprovedTimeWindowedKStream windowedBy(Windows windows); + + @Override + ImprovedTimeWindowedKStream windowedBy(SlidingWindows windows); + + @Override + ImprovedSessionWindowedKStream windowedBy(SessionWindows windows); + + @Override + ImprovedCogroupedKStream cogroup(Aggregator aggregator); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java new file mode 100644 index 00000000..4cd25946 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -0,0 +1,125 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.SlidingWindows; +import org.apache.kafka.streams.kstream.Window; +import org.apache.kafka.streams.kstream.Windows; +import org.apache.kafka.streams.state.KeyValueStore; + +@RequiredArgsConstructor +class ImprovedKGroupedStreamImpl implements ImprovedKGroupedStream { + + private final @NonNull KGroupedStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable count() { + return this.context.newTable(this.wrapped.count()); + } + + @Override + public ImprovedKTable count(final Named named) { + return this.context.newTable(this.wrapped.count(named)); + } + + @Override + public ImprovedKTable count(final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(materialized)); + } + + @Override + public ImprovedKTable count(final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(named, materialized)); + } + + @Override + public ImprovedKTable reduce(final Reducer reducer) { + return this.context.newTable(this.wrapped.reduce(reducer)); + } + + @Override + public ImprovedKTable reduce(final Reducer reducer, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(reducer, materialized)); + } + + @Override + public ImprovedKTable reduce(final Reducer reducer, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(reducer, named, materialized)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator aggregator) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator aggregator, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, materialized)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator aggregator, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, named, materialized)); + } + + @Override + public ImprovedTimeWindowedKStream windowedBy(final Windows windows) { + return this.context.newTimeWindowedStream(this.wrapped.windowedBy(windows)); + } + + @Override + public ImprovedTimeWindowedKStream windowedBy(final SlidingWindows windows) { + return this.context.newTimeWindowedStream(this.wrapped.windowedBy(windows)); + } + + @Override + public ImprovedSessionWindowedKStream windowedBy(final SessionWindows windows) { + return this.context.newSessionWindowedStream(this.wrapped.windowedBy(windows)); + } + + @Override + public ImprovedCogroupedKStream cogroup(final Aggregator aggregator) { + return this.context.newCogroupedStream(this.wrapped.cogroup(aggregator)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java new file mode 100644 index 00000000..e4d8e23a --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.KGroupedTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.state.KeyValueStore; + +public interface ImprovedKGroupedTable extends KGroupedTable { + + @Override + ImprovedKTable count(Materialized> materialized); + + @Override + ImprovedKTable count(Named named, Materialized> materialized); + + @Override + ImprovedKTable count(); + + @Override + ImprovedKTable count(Named named); + + @Override + ImprovedKTable reduce(Reducer adder, Reducer subtractor, + Materialized> materialized); + + @Override + ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, + Materialized> materialized); + + @Override + ImprovedKTable reduce(Reducer adder, Reducer subtractor); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + Aggregator subtractor, + Materialized> materialized); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + Aggregator subtractor, Named named, + Materialized> materialized); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + Aggregator subtractor); + + @Override + ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + Aggregator subtractor, Named named); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java new file mode 100644 index 00000000..a1a264a7 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java @@ -0,0 +1,111 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.KGroupedTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.state.KeyValueStore; + +@RequiredArgsConstructor +class ImprovedKGroupedTableImpl implements ImprovedKGroupedTable { + + private final @NonNull KGroupedTable wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable count(final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(materialized)); + } + + @Override + public ImprovedKTable count(final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(named, materialized)); + } + + @Override + public ImprovedKTable count() { + return this.context.newTable(this.wrapped.count()); + } + + @Override + public ImprovedKTable count(final Named named) { + return this.context.newTable(this.wrapped.count(named)); + } + + @Override + public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(adder, subtractor, materialized)); + } + + @Override + public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(adder, subtractor, materialized)); + } + + @Override + public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor) { + return this.context.newTable(this.wrapped.reduce(adder, subtractor)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator adder, + final Aggregator subtractor, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator adder, + final Aggregator subtractor, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator adder, + final Aggregator subtractor) { + return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor)); + } + + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator adder, + final Aggregator subtractor, final Named named) { + return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor, named)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java new file mode 100644 index 00000000..be1a8c7f --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -0,0 +1,391 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.ForeachAction; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.Joined; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.apache.kafka.streams.kstream.TransformerSupplier; +import org.apache.kafka.streams.kstream.ValueJoiner; +import org.apache.kafka.streams.kstream.ValueJoinerWithKey; +import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerSupplier; +import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; +import org.apache.kafka.streams.state.KeyValueStore; + +public interface ImprovedKStream extends KStream { + + @Override + ImprovedKStream filter(Predicate predicate); + + @Override + ImprovedKStream filter(Predicate predicate, Named named); + + @Override + ImprovedKStream filterNot(Predicate predicate); + + @Override + ImprovedKStream filterNot(Predicate predicate, Named named); + + @Override + ImprovedKStream selectKey(KeyValueMapper mapper); + + @Override + ImprovedKStream selectKey(KeyValueMapper mapper, Named named); + + @Override + ImprovedKStream map( + KeyValueMapper> mapper); + + @Override + ImprovedKStream map( + KeyValueMapper> mapper, Named named); + + @Override + ImprovedKStream mapValues(ValueMapper mapper); + + @Override + ImprovedKStream mapValues(ValueMapper mapper, Named named); + + @Override + ImprovedKStream mapValues(ValueMapperWithKey mapper); + + @Override + ImprovedKStream mapValues(ValueMapperWithKey mapper, Named named); + + @Override + ImprovedKStream flatMap( + KeyValueMapper>> mapper); + + @Override + ImprovedKStream flatMap( + KeyValueMapper>> mapper, + Named named); + + @Override + ImprovedKStream flatMapValues(ValueMapper> mapper); + + @Override + ImprovedKStream flatMapValues(ValueMapper> mapper, + Named named); + + @Override + ImprovedKStream flatMapValues( + ValueMapperWithKey> mapper); + + @Override + ImprovedKStream flatMapValues( + ValueMapperWithKey> mapper, + Named named); + + @Override + ImprovedKStream peek(ForeachAction action); + + @Override + ImprovedKStream peek(ForeachAction action, Named named); + + @Override + ImprovedKStream merge(KStream stream); + + @Override + ImprovedKStream merge(KStream stream, Named named); + + @Override + ImprovedKStream through(String topic); + + @Override + ImprovedKStream through(String topic, Produced produced); + + @Override + ImprovedKStream repartition(); + + @Override + ImprovedKStream repartition(Repartitioned repartitioned); + + void toOutputTopic(); + + void toOutputTopic(Produced produced); + + void toOutputTopic(ConfiguredProduced produced); + + void toOutputTopic(String label); + + void toOutputTopic(String label, Produced produced); + + void toOutputTopic(String label, ConfiguredProduced produced); + + void toErrorTopic(); + + void toErrorTopic(Produced produced); + + void toErrorTopic(ConfiguredProduced produced); + + @Override + ImprovedKTable toTable(); + + @Override + ImprovedKTable toTable(Named named); + + @Override + ImprovedKTable toTable(Materialized> materialized); + + @Override + ImprovedKTable toTable(Named named, Materialized> materialized); + + @Override + ImprovedKGroupedStream groupBy(KeyValueMapper keySelector); + + @Override + ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, + Grouped grouped); + + @Override + ImprovedKGroupedStream groupByKey(); + + @Override + ImprovedKGroupedStream groupByKey(Grouped grouped); + + @Override + ImprovedKStream join(KStream otherStream, + ValueJoiner joiner, + JoinWindows windows); + + @Override + ImprovedKStream join(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows); + + @Override + ImprovedKStream join(KStream otherStream, + ValueJoiner joiner, + JoinWindows windows, StreamJoined streamJoined); + + @Override + ImprovedKStream join(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows, + StreamJoined streamJoined); + + @Override + ImprovedKStream leftJoin(KStream otherStream, + ValueJoiner joiner, JoinWindows windows); + + @Override + ImprovedKStream leftJoin(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows); + + @Override + ImprovedKStream leftJoin(KStream otherStream, + ValueJoiner joiner, JoinWindows windows, + StreamJoined streamJoined); + + @Override + ImprovedKStream leftJoin(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows, + StreamJoined streamJoined); + + @Override + ImprovedKStream outerJoin(KStream otherStream, + ValueJoiner joiner, JoinWindows windows); + + @Override + ImprovedKStream outerJoin(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows); + + @Override + ImprovedKStream outerJoin(KStream otherStream, + ValueJoiner joiner, JoinWindows windows, + StreamJoined streamJoined); + + @Override + ImprovedKStream outerJoin(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows, + StreamJoined streamJoined); + + @Override + ImprovedKStream join(KTable table, ValueJoiner joiner); + + @Override + ImprovedKStream join(KTable table, + ValueJoinerWithKey joiner); + + @Override + ImprovedKStream join(KTable table, ValueJoiner joiner, + Joined joined); + + @Override + ImprovedKStream join(KTable table, + ValueJoinerWithKey joiner, Joined joined); + + @Override + ImprovedKStream leftJoin(KTable table, + ValueJoiner joiner); + + @Override + ImprovedKStream leftJoin(KTable table, + ValueJoinerWithKey joiner); + + @Override + ImprovedKStream leftJoin(KTable table, + ValueJoiner joiner, + Joined joined); + + @Override + ImprovedKStream leftJoin(KTable table, + ValueJoinerWithKey joiner, Joined joined); + + @Override + ImprovedKStream join(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoiner joiner); + + @Override + ImprovedKStream join(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoinerWithKey joiner); + + @Override + ImprovedKStream join(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoiner joiner, Named named); + + @Override + ImprovedKStream join(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoinerWithKey joiner, Named named); + + @Override + ImprovedKStream leftJoin(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoiner valueJoiner); + + @Override + ImprovedKStream leftJoin(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoinerWithKey valueJoiner); + + @Override + ImprovedKStream leftJoin(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoiner valueJoiner, Named named); + + @Override + ImprovedKStream leftJoin(GlobalKTable globalTable, + KeyValueMapper keySelector, + ValueJoinerWithKey valueJoiner, Named named); + + @Override + ImprovedKStream transform( + TransformerSupplier> transformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream transform( + TransformerSupplier> transformerSupplier, + Named named, String... stateStoreNames); + + @Override + ImprovedKStream flatTransform( + TransformerSupplier>> transformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream flatTransform( + TransformerSupplier>> transformerSupplier, Named named, + String... stateStoreNames); + + @Override + ImprovedKStream transformValues( + ValueTransformerSupplier valueTransformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream transformValues( + ValueTransformerSupplier valueTransformerSupplier, + Named named, String... stateStoreNames); + + @Override + ImprovedKStream transformValues( + ValueTransformerWithKeySupplier valueTransformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream transformValues( + ValueTransformerWithKeySupplier valueTransformerSupplier, Named named, + String... stateStoreNames); + + @Override + ImprovedKStream flatTransformValues( + ValueTransformerSupplier> valueTransformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream flatTransformValues( + ValueTransformerSupplier> valueTransformerSupplier, + Named named, String... stateStoreNames); + + @Override + ImprovedKStream flatTransformValues( + ValueTransformerWithKeySupplier> valueTransformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream flatTransformValues( + ValueTransformerWithKeySupplier> valueTransformerSupplier, Named named, + String... stateStoreNames); + + @Override + ImprovedKStream process( + org.apache.kafka.streams.processor.api.ProcessorSupplier processorSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream process( + org.apache.kafka.streams.processor.api.ProcessorSupplier processorSupplier, + Named named, String... stateStoreNames); + + @Override + ImprovedKStream processValues( + FixedKeyProcessorSupplier processorSupplier, + String... stateStoreNames); + + @Override + ImprovedKStream processValues( + FixedKeyProcessorSupplier processorSupplier, + Named named, String... stateStoreNames); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java new file mode 100644 index 00000000..33150b8f --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -0,0 +1,670 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.BranchedKStream; +import org.apache.kafka.streams.kstream.ForeachAction; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.Joined; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Printed; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.apache.kafka.streams.kstream.TransformerSupplier; +import org.apache.kafka.streams.kstream.ValueJoiner; +import org.apache.kafka.streams.kstream.ValueJoinerWithKey; +import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerSupplier; +import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.processor.TopicNameExtractor; +import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.state.KeyValueStore; + +@RequiredArgsConstructor +class ImprovedKStreamImpl implements ImprovedKStream { + + private final @NonNull KStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKStream filter(final Predicate predicate) { + return this.context.newStream(this.wrapped.filter(predicate)); + } + + @Override + public ImprovedKStream filter(final Predicate predicate, final Named named) { + return this.context.newStream(this.wrapped.filter(predicate, named)); + } + + @Override + public ImprovedKStream filterNot(final Predicate predicate) { + return this.context.newStream(this.wrapped.filterNot(predicate)); + } + + @Override + public ImprovedKStream filterNot(final Predicate predicate, final Named named) { + return this.context.newStream(this.wrapped.filterNot(predicate, named)); + } + + @Override + public ImprovedKStream selectKey(final KeyValueMapper mapper) { + return this.context.newStream(this.wrapped.selectKey(mapper)); + } + + @Override + public ImprovedKStream selectKey(final KeyValueMapper mapper, + final Named named) { + return this.context.newStream(this.wrapped.selectKey(mapper, named)); + } + + @Override + public ImprovedKStream map( + final KeyValueMapper> mapper) { + return this.context.newStream(this.wrapped.map(mapper)); + } + + @Override + public ImprovedKStream map( + final KeyValueMapper> mapper, + final Named named) { + return this.context.newStream(this.wrapped.map(mapper, named)); + } + + @Override + public ImprovedKStream mapValues(final ValueMapper mapper) { + return this.context.newStream(this.wrapped.mapValues(mapper)); + } + + @Override + public ImprovedKStream mapValues(final ValueMapper mapper, final Named named) { + return this.context.newStream(this.wrapped.mapValues(mapper, named)); + } + + @Override + public ImprovedKStream mapValues(final ValueMapperWithKey mapper) { + return this.context.newStream(this.wrapped.mapValues(mapper)); + } + + @Override + public ImprovedKStream mapValues(final ValueMapperWithKey mapper, + final Named named) { + return this.context.newStream(this.wrapped.mapValues(mapper, named)); + } + + @Override + public ImprovedKStream flatMap( + final KeyValueMapper>> mapper) { + return this.context.newStream(this.wrapped.flatMap(mapper)); + } + + @Override + public ImprovedKStream flatMap( + final KeyValueMapper>> mapper, + final Named named) { + return this.context.newStream(this.wrapped.flatMap(mapper, named)); + } + + @Override + public ImprovedKStream flatMapValues( + final ValueMapper> mapper) { + return this.context.newStream(this.wrapped.flatMapValues(mapper)); + } + + @Override + public ImprovedKStream flatMapValues( + final ValueMapper> mapper, + final Named named) { + return this.context.newStream(this.wrapped.flatMapValues(mapper, named)); + } + + @Override + public ImprovedKStream flatMapValues( + final ValueMapperWithKey> mapper) { + return this.context.newStream(this.wrapped.flatMapValues(mapper)); + } + + @Override + public ImprovedKStream flatMapValues( + final ValueMapperWithKey> mapper, + final Named named) { + return this.context.newStream(this.wrapped.flatMapValues(mapper, named)); + } + + @Override + public void print(final Printed printed) { + this.wrapped.print(printed); + } + + @Override + public void foreach(final ForeachAction action) { + this.wrapped.foreach(action); + } + + @Override + public void foreach(final ForeachAction action, final Named named) { + this.wrapped.foreach(action, named); + } + + @Override + public ImprovedKStream peek(final ForeachAction action) { + return this.context.newStream(this.wrapped.peek(action)); + } + + @Override + public ImprovedKStream peek(final ForeachAction action, final Named named) { + return this.context.newStream(this.wrapped.peek(action, named)); + } + + @Override + public KStream[] branch(final Predicate... predicates) { + return this.wrapped.branch(predicates); + } + + @Override + public KStream[] branch(final Named named, final Predicate... predicates) { + return this.wrapped.branch(named, predicates); + } + + @Override + public BranchedKStream split() { + return this.wrapped.split(); + } + + @Override + public BranchedKStream split(final Named named) { + return this.wrapped.split(named); + } + + @Override + public ImprovedKStream merge(final KStream stream) { + return this.context.newStream(this.wrapped.merge(stream)); + } + + @Override + public ImprovedKStream merge(final KStream stream, final Named named) { + return this.context.newStream(this.wrapped.merge(stream, named)); + } + + @Override + public ImprovedKStream through(final String topic) { + return this.context.newStream(this.wrapped.through(topic)); + } + + @Override + public ImprovedKStream through(final String topic, final Produced produced) { + return this.context.newStream(this.wrapped.through(topic, produced)); + } + + @Override + public ImprovedKStream repartition() { + return this.context.newStream(this.wrapped.repartition()); + } + + @Override + public ImprovedKStream repartition(final Repartitioned repartitioned) { + return this.context.newStream(this.wrapped.repartition(repartitioned)); + } + + @Override + public void to(final String topic) { + this.wrapped.to(topic); + } + + @Override + public void to(final String topic, final Produced produced) { + this.wrapped.to(topic, produced); + } + + @Override + public void to(final TopicNameExtractor topicExtractor) { + this.wrapped.to(topicExtractor); + } + + @Override + public void to(final TopicNameExtractor topicExtractor, final Produced produced) { + this.wrapped.to(topicExtractor, produced); + } + + @Override + public void toOutputTopic() { + this.to(this.context.getTopics().getOutputTopic()); + } + + @Override + public void toOutputTopic(final Produced produced) { + this.to(this.context.getTopics().getOutputTopic(), produced); + } + + @Override + public void toOutputTopic(final ConfiguredProduced produced) { + this.toOutputTopic(produced.configure(this.context.getConfigurator())); + } + + @Override + public void toOutputTopic(final String label) { + this.to(this.context.getTopics().getOutputTopic(label)); + } + + @Override + public void toOutputTopic(final String label, final Produced produced) { + this.to(this.context.getTopics().getOutputTopic(label), produced); + } + + @Override + public void toOutputTopic(final String label, final ConfiguredProduced produced) { + this.toOutputTopic(label, produced.configure(this.context.getConfigurator())); + } + + @Override + public void toErrorTopic() { + this.to(this.context.getTopics().getErrorTopic()); + } + + @Override + public void toErrorTopic(final Produced produced) { + this.to(this.context.getTopics().getErrorTopic(), produced); + } + + @Override + public void toErrorTopic(final ConfiguredProduced produced) { + this.toErrorTopic(produced.configure(this.context.getConfigurator())); + } + + @Override + public ImprovedKTable toTable() { + return this.context.newTable(this.wrapped.toTable()); + } + + @Override + public ImprovedKTable toTable(final Named named) { + return this.context.newTable(this.wrapped.toTable(named)); + } + + @Override + public ImprovedKTable toTable(final Materialized> materialized) { + return this.context.newTable(this.wrapped.toTable(materialized)); + } + + @Override + public ImprovedKTable toTable(final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.toTable(named, materialized)); + } + + @Override + public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector) { + return this.context.newGroupedStream(this.wrapped.groupBy(keySelector)); + } + + @Override + public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector, + final Grouped grouped) { + return this.context.newGroupedStream(this.wrapped.groupBy(keySelector, grouped)); + } + + @Override + public ImprovedKGroupedStream groupByKey() { + return this.context.newGroupedStream(this.wrapped.groupByKey()); + } + + @Override + public ImprovedKGroupedStream groupByKey(final Grouped grouped) { + return this.context.newGroupedStream(this.wrapped.groupByKey(grouped)); + } + + @Override + public ImprovedKStream join(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows) { + return this.context.newStream(this.wrapped.join(otherStream, joiner, windows)); + } + + @Override + public ImprovedKStream join(final KStream otherStream, + final ValueJoinerWithKey joiner, + final JoinWindows windows) { + return this.context.newStream(this.wrapped.join(otherStream, joiner, windows)); + } + + @Override + public ImprovedKStream join(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows, + final StreamJoined streamJoined) { + return this.context.newStream(this.wrapped.join(otherStream, joiner, windows, streamJoined)); + } + + @Override + public ImprovedKStream join(final KStream otherStream, + final ValueJoinerWithKey joiner, final JoinWindows windows, + final StreamJoined streamJoined) { + return this.context.newStream(this.wrapped.join(otherStream, joiner, windows, streamJoined)); + } + + @Override + public ImprovedKStream leftJoin(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows) { + return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows)); + } + + @Override + public ImprovedKStream leftJoin(final KStream otherStream, + final ValueJoinerWithKey joiner, + final JoinWindows windows) { + return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows)); + } + + @Override + public ImprovedKStream leftJoin(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows, + final StreamJoined streamJoined) { + return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); + } + + @Override + public ImprovedKStream leftJoin(final KStream otherStream, + final ValueJoinerWithKey joiner, final JoinWindows windows, + final StreamJoined streamJoined) { + return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); + } + + @Override + public ImprovedKStream outerJoin(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows) { + return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows)); + } + + @Override + public ImprovedKStream outerJoin(final KStream otherStream, + final ValueJoinerWithKey joiner, + final JoinWindows windows) { + return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows)); + } + + @Override + public ImprovedKStream outerJoin(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows, + final StreamJoined streamJoined) { + return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); + } + + @Override + public ImprovedKStream outerJoin(final KStream otherStream, + final ValueJoinerWithKey joiner, final JoinWindows windows, + final StreamJoined streamJoined) { + return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); + } + + @Override + public ImprovedKStream join(final KTable table, + final ValueJoiner joiner) { + return this.context.newStream(this.wrapped.join(table, joiner)); + } + + @Override + public ImprovedKStream join(final KTable table, + final ValueJoinerWithKey joiner) { + return this.context.newStream(this.wrapped.join(table, joiner)); + } + + @Override + public ImprovedKStream join(final KTable table, + final ValueJoiner joiner, final Joined joined) { + return this.context.newStream(this.wrapped.join(table, joiner, joined)); + } + + @Override + public ImprovedKStream join(final KTable table, + final ValueJoinerWithKey joiner, + final Joined joined) { + return this.context.newStream(this.wrapped.join(table, joiner, joined)); + } + + @Override + public ImprovedKStream leftJoin(final KTable table, + final ValueJoiner joiner) { + return this.context.newStream(this.wrapped.leftJoin(table, joiner)); + } + + @Override + public ImprovedKStream leftJoin(final KTable table, + final ValueJoinerWithKey joiner) { + return this.context.newStream(this.wrapped.leftJoin(table, joiner)); + } + + @Override + public ImprovedKStream leftJoin(final KTable table, + final ValueJoiner joiner, final Joined joined) { + return this.context.newStream(this.wrapped.leftJoin(table, joiner, joined)); + } + + @Override + public ImprovedKStream leftJoin(final KTable table, + final ValueJoinerWithKey joiner, + final Joined joined) { + return this.context.newStream(this.wrapped.leftJoin(table, joiner, joined)); + } + + @Override + public ImprovedKStream join(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoiner joiner) { + return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner)); + } + + @Override + public ImprovedKStream join(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoinerWithKey joiner) { + return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner)); + } + + @Override + public ImprovedKStream join(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoiner joiner, final Named named) { + return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner, named)); + } + + @Override + public ImprovedKStream join(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoinerWithKey joiner, final Named named) { + return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner, named)); + } + + @Override + public ImprovedKStream leftJoin(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoiner valueJoiner) { + return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); + } + + @Override + public ImprovedKStream leftJoin(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoinerWithKey valueJoiner) { + return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); + } + + @Override + public ImprovedKStream leftJoin(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoiner valueJoiner, final Named named) { + return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); + } + + @Override + public ImprovedKStream leftJoin(final GlobalKTable globalTable, + final KeyValueMapper keySelector, + final ValueJoinerWithKey valueJoiner, final Named named) { + return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); + } + + @Override + public ImprovedKStream transform( + final TransformerSupplier> transformerSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.transform(transformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream transform( + final TransformerSupplier> transformerSupplier, final Named named, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.transform(transformerSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKStream flatTransform( + final TransformerSupplier>> transformerSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.flatTransform(transformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream flatTransform( + final TransformerSupplier>> transformerSupplier, + final Named named, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.flatTransform(transformerSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKStream transformValues( + final ValueTransformerSupplier valueTransformerSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream transformValues( + final ValueTransformerSupplier valueTransformerSupplier, final Named named, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKStream transformValues( + final ValueTransformerWithKeySupplier valueTransformerSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream transformValues( + final ValueTransformerWithKeySupplier valueTransformerSupplier, + final Named named, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKStream flatTransformValues( + final ValueTransformerSupplier> valueTransformerSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream flatTransformValues( + final ValueTransformerSupplier> valueTransformerSupplier, final Named named, + final String... stateStoreNames) { + return this.context.newStream( + this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKStream flatTransformValues( + final ValueTransformerWithKeySupplier> valueTransformerSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream flatTransformValues( + final ValueTransformerWithKeySupplier> valueTransformerSupplier, + final Named named, + final String... stateStoreNames) { + return this.context.newStream( + this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); + } + + @Override + public void process( + final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier, + final String... stateStoreNames) { + this.wrapped.process(processorSupplier, stateStoreNames); + } + + @Override + public void process( + final org.apache.kafka.streams.processor.ProcessorSupplier processorSupplier, + final Named named, final String... stateStoreNames) { + this.wrapped.process(processorSupplier, named, stateStoreNames); + } + + @Override + public ImprovedKStream process( + final ProcessorSupplier processorSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.process(processorSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream process( + final ProcessorSupplier processorSupplier, final Named named, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.process(processorSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKStream processValues( + final FixedKeyProcessorSupplier processorSupplier, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.processValues(processorSupplier, stateStoreNames)); + } + + @Override + public ImprovedKStream processValues( + final FixedKeyProcessorSupplier processorSupplier, final Named named, + final String... stateStoreNames) { + return this.context.newStream(this.wrapped.processValues(processorSupplier, named, stateStoreNames)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java new file mode 100644 index 00000000..459e1326 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -0,0 +1,246 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Function; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Suppressed; +import org.apache.kafka.streams.kstream.TableJoined; +import org.apache.kafka.streams.kstream.ValueJoiner; +import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.state.KeyValueStore; + +public interface ImprovedKTable extends KTable { + + @Override + ImprovedKTable filter(Predicate predicate); + + @Override + ImprovedKTable filter(Predicate predicate, Named named); + + @Override + ImprovedKTable filter(Predicate predicate, + Materialized> materialized); + + @Override + ImprovedKTable filter(Predicate predicate, Named named, + Materialized> materialized); + + @Override + ImprovedKTable filterNot(Predicate predicate); + + @Override + ImprovedKTable filterNot(Predicate predicate, Named named); + + @Override + ImprovedKTable filterNot(Predicate predicate, + Materialized> materialized); + + @Override + ImprovedKTable filterNot(Predicate predicate, Named named, + Materialized> materialized); + + @Override + ImprovedKTable mapValues(ValueMapper mapper); + + @Override + ImprovedKTable mapValues(ValueMapper mapper, Named named); + + @Override + ImprovedKTable mapValues(ValueMapperWithKey mapper); + + @Override + ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named); + + @Override + ImprovedKTable mapValues(ValueMapper mapper, + Materialized> materialized); + + @Override + ImprovedKTable mapValues(ValueMapper mapper, Named named, + Materialized> materialized); + + @Override + ImprovedKTable mapValues(ValueMapperWithKey mapper, + Materialized> materialized); + + @Override + ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, + Materialized> materialized); + + @Override + ImprovedKStream toStream(); + + @Override + ImprovedKStream toStream(Named named); + + @Override + ImprovedKStream toStream(KeyValueMapper mapper); + + @Override + ImprovedKStream toStream(KeyValueMapper mapper, Named named); + + @Override + ImprovedKTable suppress(Suppressed suppressed); + + @Override + ImprovedKTable transformValues( + ValueTransformerWithKeySupplier transformerSupplier, + String... stateStoreNames); + + @Override + ImprovedKTable transformValues( + ValueTransformerWithKeySupplier transformerSupplier, Named named, + String... stateStoreNames); + + @Override + ImprovedKTable transformValues( + ValueTransformerWithKeySupplier transformerSupplier, + Materialized> materialized, String... stateStoreNames); + + @Override + ImprovedKTable transformValues( + ValueTransformerWithKeySupplier transformerSupplier, + Materialized> materialized, Named named, String... stateStoreNames); + + @Override + ImprovedKGroupedTable groupBy(KeyValueMapper> selector); + + @Override + ImprovedKGroupedTable groupBy(KeyValueMapper> selector, + Grouped grouped); + + @Override + ImprovedKTable join(KTable other, ValueJoiner joiner); + + @Override + ImprovedKTable join(KTable other, ValueJoiner joiner, + Named named); + + @Override + ImprovedKTable join(KTable other, ValueJoiner joiner, + Materialized> materialized); + + @Override + ImprovedKTable join(KTable other, ValueJoiner joiner, + Named named, Materialized> materialized); + + @Override + ImprovedKTable leftJoin(KTable other, + ValueJoiner joiner); + + @Override + ImprovedKTable leftJoin(KTable other, + ValueJoiner joiner, + Named named); + + @Override + ImprovedKTable leftJoin(KTable other, + ValueJoiner joiner, + Materialized> materialized); + + @Override + ImprovedKTable leftJoin(KTable other, + ValueJoiner joiner, + Named named, Materialized> materialized); + + @Override + ImprovedKTable outerJoin(KTable other, + ValueJoiner joiner); + + @Override + ImprovedKTable outerJoin(KTable other, + ValueJoiner joiner, + Named named); + + @Override + ImprovedKTable outerJoin(KTable other, + ValueJoiner joiner, + Materialized> materialized); + + @Override + ImprovedKTable outerJoin(KTable other, + ValueJoiner joiner, + Named named, Materialized> materialized); + + @Override + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner); + + @Override + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Named named); + + @Override + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined); + + @Override + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Materialized> materialized); + + @Override + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Named named, Materialized> materialized); + + @Override + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + Materialized> materialized); + + @Override + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner); + + @Override + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Named named); + + @Override + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined); + + @Override + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Materialized> materialized); + + @Override + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Named named, Materialized> materialized); + + @Override + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + Materialized> materialized); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java new file mode 100644 index 00000000..ae170737 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -0,0 +1,381 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Function; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Grouped; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Suppressed; +import org.apache.kafka.streams.kstream.TableJoined; +import org.apache.kafka.streams.kstream.ValueJoiner; +import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.state.KeyValueStore; + +@RequiredArgsConstructor +class ImprovedKTableImpl implements ImprovedKTable { + + private final @NonNull KTable wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable filter(final Predicate predicate) { + return this.context.newTable(this.wrapped.filter(predicate)); + } + + @Override + public ImprovedKTable filter(final Predicate predicate, final Named named) { + return this.context.newTable(this.wrapped.filter(predicate, named)); + } + + @Override + public ImprovedKTable filter(final Predicate predicate, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.filter(predicate, materialized)); + } + + @Override + public ImprovedKTable filter(final Predicate predicate, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.filter(predicate, named, materialized)); + } + + @Override + public ImprovedKTable filterNot(final Predicate predicate) { + return this.context.newTable(this.wrapped.filterNot(predicate)); + } + + @Override + public ImprovedKTable filterNot(final Predicate predicate, final Named named) { + return this.context.newTable(this.wrapped.filterNot(predicate, named)); + } + + @Override + public ImprovedKTable filterNot(final Predicate predicate, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.filterNot(predicate, materialized)); + } + + @Override + public ImprovedKTable filterNot(final Predicate predicate, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.filterNot(predicate, materialized)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapper mapper) { + return this.context.newTable(this.wrapped.mapValues(mapper)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapper mapper, final Named named) { + return this.context.newTable(this.wrapped.mapValues(mapper, named)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapperWithKey mapper) { + return this.context.newTable(this.wrapped.mapValues(mapper)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + final Named named) { + return this.context.newTable(this.wrapped.mapValues(mapper, named)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapper mapper, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.mapValues(mapper, materialized)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapper mapper, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.mapValues(mapper, named, materialized)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.mapValues(mapper, materialized)); + } + + @Override + public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + final Named named, final Materialized> materialized) { + return this.context.newTable(this.wrapped.mapValues(mapper, named, materialized)); + } + + @Override + public ImprovedKStream toStream() { + return this.context.newStream(this.wrapped.toStream()); + } + + @Override + public ImprovedKStream toStream(final Named named) { + return this.context.newStream(this.wrapped.toStream(named)); + } + + @Override + public ImprovedKStream toStream(final KeyValueMapper mapper) { + return this.context.newStream(this.wrapped.toStream(mapper)); + } + + @Override + public ImprovedKStream toStream(final KeyValueMapper mapper, + final Named named) { + return this.context.newStream(this.wrapped.toStream(mapper, named)); + } + + @Override + public ImprovedKTable suppress(final Suppressed suppressed) { + return this.context.newTable(this.wrapped.suppress(suppressed)); + } + + @Override + public ImprovedKTable transformValues( + final ValueTransformerWithKeySupplier transformerSupplier, + final String... stateStoreNames) { + return this.context.newTable(this.wrapped.transformValues(transformerSupplier, stateStoreNames)); + } + + @Override + public ImprovedKTable transformValues( + final ValueTransformerWithKeySupplier transformerSupplier, + final Named named, + final String... stateStoreNames) { + return this.context.newTable(this.wrapped.transformValues(transformerSupplier, named, stateStoreNames)); + } + + @Override + public ImprovedKTable transformValues( + final ValueTransformerWithKeySupplier transformerSupplier, + final Materialized> materialized, final String... stateStoreNames) { + return this.context.newTable(this.wrapped.transformValues(transformerSupplier, materialized, stateStoreNames)); + } + + @Override + public ImprovedKTable transformValues( + final ValueTransformerWithKeySupplier transformerSupplier, + final Materialized> materialized, final Named named, + final String... stateStoreNames) { + return this.context.newTable( + this.wrapped.transformValues(transformerSupplier, materialized, named, stateStoreNames)); + } + + @Override + public ImprovedKGroupedTable groupBy( + final KeyValueMapper> selector) { + return this.context.newGroupedTable(this.wrapped.groupBy(selector)); + } + + @Override + public ImprovedKGroupedTable groupBy( + final KeyValueMapper> selector, final Grouped grouped) { + return this.context.newGroupedTable(this.wrapped.groupBy(selector, grouped)); + } + + @Override + public ImprovedKTable join(final KTable other, + final ValueJoiner joiner) { + return this.context.newTable(this.wrapped.join(other, joiner)); + } + + @Override + public ImprovedKTable join(final KTable other, + final ValueJoiner joiner, final Named named) { + return this.context.newTable(this.wrapped.join(other, joiner, named)); + } + + @Override + public ImprovedKTable join(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.join(other, joiner, materialized)); + } + + @Override + public ImprovedKTable join(final KTable other, + final ValueJoiner joiner, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.join(other, joiner, materialized)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final ValueJoiner joiner) { + return this.context.newTable(this.wrapped.leftJoin(other, joiner)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final ValueJoiner joiner, final Named named) { + return this.context.newTable(this.wrapped.leftJoin(other, joiner, named)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.leftJoin(other, joiner, materialized)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final ValueJoiner joiner, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.leftJoin(other, joiner, materialized)); + } + + @Override + public ImprovedKTable outerJoin(final KTable other, + final ValueJoiner joiner) { + return this.context.newTable(this.wrapped.outerJoin(other, joiner)); + } + + @Override + public ImprovedKTable outerJoin(final KTable other, + final ValueJoiner joiner, final Named named) { + return this.context.newTable(this.wrapped.outerJoin(other, joiner, named)); + } + + @Override + public ImprovedKTable outerJoin(final KTable other, + final ValueJoiner joiner, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.outerJoin(other, joiner, materialized)); + } + + @Override + public ImprovedKTable outerJoin(final KTable other, + final ValueJoiner joiner, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.outerJoin(other, joiner, materialized)); + } + + @Override + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner) { + return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner)); + } + + @Override + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final Named named) { + return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, named)); + } + + @Override + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined) { + return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined)); + } + + @Override + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final Materialized> materialized) { + return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, materialized)); + } + + @Override + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, named, materialized)); + } + + @Override + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined, materialized)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner) { + return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final Named named) { + return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined) { + return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final Materialized> materialized) { + return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, materialized)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named, materialized)); + } + + @Override + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final Materialized> materialized) { + return this.context.newTable( + this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, materialized)); + } + + @Override + public String queryableStoreName() { + return this.wrapped.queryableStoreName(); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java new file mode 100644 index 00000000..19a4c199 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -0,0 +1,52 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Merger; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.SessionStore; + +public interface ImprovedSessionWindowedCogroupedKStream extends SessionWindowedCogroupedKStream { + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger); + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + Named named); + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + Materialized> materialized); + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + Named named, Materialized> materialized); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java new file mode 100644 index 00000000..7194695a --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java @@ -0,0 +1,70 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Merger; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.SessionStore; + +@RequiredArgsConstructor +class ImprovedSessionWindowedCogroupedStreamImpl implements ImprovedSessionWindowedCogroupedKStream { + + private final @NonNull SessionWindowedCogroupedKStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Merger sessionMerger) { + return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger)); + } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Merger sessionMerger, + final Named named) { + return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger, named)); + } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Merger sessionMerger, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger, materialized)); + } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Merger sessionMerger, + final Named named, final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger, materialized)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java new file mode 100644 index 00000000..83ed6b08 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -0,0 +1,91 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.EmitStrategy; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Merger; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.kstream.SessionWindowedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.SessionStore; + +public interface ImprovedSessionWindowedKStream extends SessionWindowedKStream { + + @Override + ImprovedKTable, Long> count(); + + @Override + ImprovedKTable, Long> count(Named named); + + @Override + ImprovedKTable, Long> count(Materialized> materialized); + + @Override + ImprovedKTable, Long> count(Named named, + Materialized> materialized); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Merger sessionMerger); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Merger sessionMerger, Named named); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Merger sessionMerger, Materialized> materialized); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Merger sessionMerger, Named named, + Materialized> materialized); + + @Override + ImprovedKTable, V> reduce(Reducer reducer); + + @Override + ImprovedKTable, V> reduce(Reducer reducer, Named named); + + @Override + ImprovedKTable, V> reduce(Reducer reducer, + Materialized> materialized); + + @Override + ImprovedKTable, V> reduce(Reducer reducer, Named named, + Materialized> materialized); + + @Override + ImprovedSessionWindowedKStream emitStrategy(EmitStrategy emitStrategy); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java new file mode 100644 index 00000000..e461fd00 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java @@ -0,0 +1,123 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.EmitStrategy; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Merger; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.kstream.SessionWindowedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.SessionStore; + +@RequiredArgsConstructor +class ImprovedSessionWindowedStreamImpl implements ImprovedSessionWindowedKStream { + + private final @NonNull SessionWindowedKStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable, Long> count() { + return this.context.newTable(this.wrapped.count()); + } + + @Override + public ImprovedKTable, Long> count(final Named named) { + return this.context.newTable(this.wrapped.count(named)); + } + + @Override + public ImprovedKTable, Long> count( + final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(materialized)); + } + + @Override + public ImprovedKTable, Long> count(final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(named, materialized)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Merger sessionMerger) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Merger sessionMerger, + final Named named) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger, named)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Merger sessionMerger, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Merger sessionMerger, + final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer) { + return this.context.newTable(this.wrapped.reduce(reducer)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, final Named named) { + return this.context.newTable(this.wrapped.reduce(reducer, named)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(reducer, materialized)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(reducer, named, materialized)); + } + + @Override + public ImprovedSessionWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { + return this.context.newSessionWindowedStream(this.wrapped.emitStrategy(emitStrategy)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java new file mode 100644 index 00000000..89a12ae1 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -0,0 +1,50 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.TimeWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.WindowStore; + +public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindowedCogroupedKStream { + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer); + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, Named named); + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, + Materialized> materialized); + + @Override + ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, + Materialized> materialized); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java new file mode 100644 index 00000000..e8676636 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java @@ -0,0 +1,64 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.TimeWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.WindowStore; + +@RequiredArgsConstructor +class ImprovedTimeWindowedCogroupedStreamImpl implements ImprovedTimeWindowedCogroupedKStream { + + private final @NonNull TimeWindowedCogroupedKStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer) { + return this.context.newTable(this.wrapped.aggregate(initializer)); + } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named) { + return this.context.newTable(this.wrapped.aggregate(initializer, named)); + } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, materialized)); + } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, named, materialized)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java new file mode 100644 index 00000000..8e60053b --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -0,0 +1,88 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.EmitStrategy; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.kstream.TimeWindowedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.WindowStore; + +public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream { + + @Override + ImprovedKTable, Long> count(); + + @Override + ImprovedKTable, Long> count(Named named); + + @Override + ImprovedKTable, Long> count(Materialized> materialized); + + @Override + ImprovedKTable, Long> count(Named named, + Materialized> materialized); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Named named); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Materialized> materialized); + + @Override + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Named named, Materialized> materialized); + + @Override + ImprovedKTable, V> reduce(Reducer reducer); + + @Override + ImprovedKTable, V> reduce(Reducer reducer, Named named); + + @Override + ImprovedKTable, V> reduce(Reducer reducer, + Materialized> materialized); + + @Override + ImprovedKTable, V> reduce(Reducer reducer, Named named, + Materialized> materialized); + + @Override + ImprovedTimeWindowedKStream emitStrategy(EmitStrategy emitStrategy); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java new file mode 100644 index 00000000..6bf04f4d --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java @@ -0,0 +1,120 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Aggregator; +import org.apache.kafka.streams.kstream.EmitStrategy; +import org.apache.kafka.streams.kstream.Initializer; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Reducer; +import org.apache.kafka.streams.kstream.TimeWindowedKStream; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.state.WindowStore; + +@RequiredArgsConstructor +class ImprovedTimeWindowedStreamImpl implements ImprovedTimeWindowedKStream { + + private final @NonNull TimeWindowedKStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedKTable, Long> count() { + return this.context.newTable(this.wrapped.count()); + } + + @Override + public ImprovedKTable, Long> count(final Named named) { + return this.context.newTable(this.wrapped.count(named)); + } + + @Override + public ImprovedKTable, Long> count( + final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(materialized)); + } + + @Override + public ImprovedKTable, Long> count(final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.count(named, materialized)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Named named) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, named)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, materialized)); + } + + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, materialized)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer) { + return this.context.newTable(this.wrapped.reduce(reducer)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, final Named named) { + return this.context.newTable(this.wrapped.reduce(reducer, named)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(reducer, materialized)); + } + + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + final Materialized> materialized) { + return this.context.newTable(this.wrapped.reduce(reducer, named, materialized)); + } + + @Override + public ImprovedTimeWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { + return this.context.newTimeWindowedStream(this.wrapped.emitStrategy(emitStrategy)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java new file mode 100644 index 00000000..9fbeab1e --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -0,0 +1,89 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NonNull; +import lombok.Value; +import org.apache.kafka.streams.kstream.CogroupedKStream; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.KGroupedTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.SessionWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.SessionWindowedKStream; +import org.apache.kafka.streams.kstream.TimeWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.TimeWindowedKStream; + +@Value +@Getter(AccessLevel.PACKAGE) +public class StreamsContext { + + @NonNull + StreamsTopicConfig topics; + @NonNull + Configurator configurator; + + public ImprovedKStream newStream(final KStream stream) { + return new ImprovedKStreamImpl<>(stream, this); + } + + public ImprovedKGroupedStream newGroupedStream(final KGroupedStream stream) { + return new ImprovedKGroupedStreamImpl<>(stream, this); + } + + public ImprovedTimeWindowedKStream newTimeWindowedStream( + final TimeWindowedKStream stream) { + return new ImprovedTimeWindowedStreamImpl<>(stream, this); + } + + public ImprovedSessionWindowedKStream newSessionWindowedStream( + final SessionWindowedKStream stream) { + return new ImprovedSessionWindowedStreamImpl<>(stream, this); + } + + public ImprovedTimeWindowedCogroupedKStream newTimeWindowedCogroupedStream( + final TimeWindowedCogroupedKStream stream) { + return new ImprovedTimeWindowedCogroupedStreamImpl<>(stream, this); + } + + public ImprovedSessionWindowedCogroupedKStream newSessionWindowedCogroupedStream( + final SessionWindowedCogroupedKStream stream) { + return new ImprovedSessionWindowedCogroupedStreamImpl<>(stream, this); + } + + public ImprovedCogroupedKStream newCogroupedStream(final CogroupedKStream stream) { + return new ImprovedCogroupedStreamImpl<>(stream, this); + } + + public ImprovedKTable newTable(final KTable table) { + return new ImprovedKTableImpl<>(table, this); + } + + public ImprovedKGroupedTable newGroupedTable(final KGroupedTable table) { + return new ImprovedKGroupedTableImpl<>(table, this); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index 13717ead..bd050372 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,14 +24,15 @@ package com.bakdata.kafka; +import java.util.Collection; import java.util.Map; +import java.util.regex.Pattern; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Value; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.KStream; /** * Provides all runtime configurations and supports building a {@link Topology} of a {@link StreamsApp} @@ -48,6 +49,41 @@ public class TopologyBuilder { @NonNull Map kafkaProperties; + public ImprovedKStream stream(final String topic) { + return this.getContext().newStream(this.streamsBuilder.stream(topic)); + } + + public ImprovedKStream stream(final String topic, final Consumed consumed) { + return this.getContext().newStream(this.streamsBuilder.stream(topic, consumed)); + } + + public ImprovedKStream stream(final Collection topics) { + return this.getContext().newStream(this.streamsBuilder.stream(topics)); + } + + public ImprovedKStream stream(final Collection topics, final Consumed consumed) { + return this.getContext().newStream(this.streamsBuilder.stream(topics, consumed)); + } + + public ImprovedKStream stream(final Pattern topicPattern) { + return this.getContext().newStream(this.streamsBuilder.stream(topicPattern)); + } + + public ImprovedKStream stream(final Pattern topicPattern, final Consumed consumed) { + return this.getContext().newStream(this.streamsBuilder.stream(topicPattern, consumed)); + } + + /** + * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics()} + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + */ + public ImprovedKStream streamInput(final Consumed consumed) { + return this.stream(this.topics.getInputTopics(), consumed); + } + /** * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics()} * @param consumed define optional parameters for streaming topics @@ -55,8 +91,8 @@ public class TopologyBuilder { * @param type of keys * @param type of values */ - public KStream streamInput(final Consumed consumed) { - return this.streamsBuilder.stream(this.topics.getInputTopics(), consumed); + public ImprovedKStream streamInput(final ConfiguredConsumed consumed) { + return this.streamInput(consumed.configure(this.createConfigurator())); } /** @@ -65,8 +101,20 @@ public KStream streamInput(final Consumed consumed) { * @param type of keys * @param type of values */ - public KStream streamInput() { - return this.streamsBuilder.stream(this.topics.getInputTopics()); + public ImprovedKStream streamInput() { + return this.stream(this.topics.getInputTopics()); + } + + /** + * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics(String)} + * @param label label of input topics + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics(String)} + * @param type of keys + * @param type of values + */ + public ImprovedKStream streamInput(final String label, final Consumed consumed) { + return this.stream(this.topics.getInputTopics(label), consumed); } /** @@ -77,8 +125,8 @@ public KStream streamInput() { * @param type of keys * @param type of values */ - public KStream streamInput(final String label, final Consumed consumed) { - return this.streamsBuilder.stream(this.topics.getInputTopics(label), consumed); + public ImprovedKStream streamInput(final String label, final ConfiguredConsumed consumed) { + return this.streamInput(label, consumed.configure(this.createConfigurator())); } /** @@ -88,8 +136,19 @@ public KStream streamInput(final String label, final Consumed * @param type of keys * @param type of values */ - public KStream streamInput(final String label) { - return this.streamsBuilder.stream(this.topics.getInputTopics(label)); + public ImprovedKStream streamInput(final String label) { + return this.stream(this.topics.getInputTopics(label)); + } + + /** + * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern()} + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} + * @param type of keys + * @param type of values + */ + public ImprovedKStream streamInputPattern(final Consumed consumed) { + return this.stream(this.topics.getInputPattern(), consumed); } /** @@ -99,8 +158,8 @@ public KStream streamInput(final String label) { * @param type of keys * @param type of values */ - public KStream streamInputPattern(final Consumed consumed) { - return this.streamsBuilder.stream(this.topics.getInputPattern(), consumed); + public ImprovedKStream streamInputPattern(final ConfiguredConsumed consumed) { + return this.streamInputPattern(consumed.configure(this.createConfigurator())); } /** @@ -109,8 +168,8 @@ public KStream streamInputPattern(final Consumed consumed) { * @param type of keys * @param type of values */ - public KStream streamInputPattern() { - return this.streamsBuilder.stream(this.topics.getInputPattern()); + public ImprovedKStream streamInputPattern() { + return this.stream(this.topics.getInputPattern()); } /** @@ -121,19 +180,32 @@ public KStream streamInputPattern() { * @param type of keys * @param type of values */ - public KStream streamInputPattern(final String label, final Consumed consumed) { - return this.streamsBuilder.stream(this.topics.getInputPattern(label), consumed); + public ImprovedKStream streamInputPattern(final String label, final Consumed consumed) { + return this.stream(this.topics.getInputPattern(label), consumed); } /** * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param label label of input pattern + * @param consumed define optional parameters for streaming topics * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values */ - public KStream streamInputPattern(final String label) { - return this.streamsBuilder.stream(this.topics.getInputPattern(label)); + public ImprovedKStream streamInputPattern(final String label, + final ConfiguredConsumed consumed) { + return this.streamInputPattern(label, consumed.configure(this.createConfigurator())); + } + + /** + * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * @param label label of input pattern + * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * @param type of keys + * @param type of values + */ + public ImprovedKStream streamInputPattern(final String label) { + return this.stream(this.topics.getInputPattern(label)); } /** @@ -153,6 +225,10 @@ public EffectiveAppConfiguration createEffectiveConfiguratio return new EffectiveAppConfiguration<>(this.topics, this.kafkaProperties); } + public StreamsContext getContext() { + return new StreamsContext(this.topics, this.createConfigurator()); + } + Topology build() { return this.streamsBuilder.build(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ExecutableStreamsAppTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ExecutableStreamsAppTest.java index f4e90d09..7f7d2f07 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ExecutableStreamsAppTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ExecutableStreamsAppTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -118,7 +118,7 @@ public StreamsCleanUpConfiguration setupCleanUp( @Override public void buildTopology(final TopologyBuilder builder) { builder.streamInput() - .to(builder.getTopics().getOutputTopic()); + .toOutputTopic(); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java index 839f05b2..1ad9dfbd 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +31,7 @@ import com.bakdata.kafka.AppConfiguration; import com.bakdata.kafka.ConfiguredStreamsApp; +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsExecutionOptions; @@ -60,7 +61,6 @@ import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler; import org.apache.kafka.streams.errors.StreamsUncaughtExceptionHandler.StreamThreadExceptionResponse; -import org.apache.kafka.streams.kstream.KStream; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; @@ -243,9 +243,9 @@ private static class ErrorApplication implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); + final ImprovedKStream input = builder.streamInput(); input.map((k, v) -> {throw new RuntimeException("Error in map");}) - .to(builder.getTopics().getOutputTopic()); + .toOutputTopic(); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java index 4f5b2680..cd001e41 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,8 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.ImprovedKTable; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -34,8 +36,6 @@ import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serdes.StringSerde; import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.TimeWindows; @@ -47,11 +47,11 @@ public class ComplexTopologyApplication implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); + final ImprovedKStream input = builder.streamInput(); input.to(THROUGH_TOPIC); - final KStream through = builder.getStreamsBuilder().stream(THROUGH_TOPIC); - final KTable, TestRecord> reduce = through + final ImprovedKStream through = builder.stream(THROUGH_TOPIC); + final ImprovedKTable, TestRecord> reduce = through .groupByKey() .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMillis(5L))) .reduce((a, b) -> a); @@ -61,7 +61,7 @@ public void buildTopology(final TopologyBuilder builder) { .groupByKey() .count(Materialized.with(Serdes.String(), Serdes.Long())) .toStream() - .to(builder.getTopics().getOutputTopic(), Produced.with(Serdes.String(), Serdes.Long())); + .toOutputTopic(Produced.with(Serdes.String(), Serdes.Long())); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java index 51087643..eee7c32d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,20 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; import com.bakdata.kafka.TopologyBuilder; import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; @NoArgsConstructor public class LabeledInputTopics implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput("label"); - input.to(builder.getTopics().getOutputTopic()); + final ImprovedKStream input = builder.streamInput("label"); + input.toOutputTopic(); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java index 54aee24c..8833fae6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,20 +24,20 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; import com.bakdata.kafka.TopologyBuilder; import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; @NoArgsConstructor public class Mirror implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); - input.to(builder.getTopics().getOutputTopic()); + final ImprovedKStream input = builder.streamInput(); + input.toOutputTopic(); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java index 6f303f2e..b2cce89b 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -32,14 +33,13 @@ import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; @NoArgsConstructor public class MirrorKeyWithAvro implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); - input.to(builder.getTopics().getOutputTopic()); + final ImprovedKStream input = builder.streamInput(); + input.toOutputTopic(); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java index fb8087d8..656c8f4d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,7 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -32,14 +33,13 @@ import io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde; import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; @NoArgsConstructor public class MirrorValueWithAvro implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream input = builder.streamInput(); - input.to(builder.getTopics().getOutputTopic()); + final ImprovedKStream input = builder.streamInput(); + input.toOutputTopic(); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java index 7406b484..87a5d051 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,7 +24,10 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.Configurator; +import com.bakdata.kafka.ConfiguredConsumed; +import com.bakdata.kafka.ConfiguredProduced; +import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.Preconfigured; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -34,9 +37,6 @@ import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.Produced; @NoArgsConstructor public class MirrorWithNonDefaultSerde implements StreamsApp { @@ -51,12 +51,11 @@ public static Serde newValueSerde() { @Override public void buildTopology(final TopologyBuilder builder) { - final Configurator configurator = builder.createConfigurator(); - final Serde valueSerde = configurator.configureForValues(newValueSerde()); - final Serde keySerde = configurator.configureForKeys(newKeySerde()); - final KStream input = - builder.streamInput(Consumed.with(keySerde, valueSerde)); - input.to(builder.getTopics().getOutputTopic(), Produced.with(keySerde, valueSerde)); + final Preconfigured> valueSerde = Preconfigured.create(newValueSerde()); + final Preconfigured> keySerde = Preconfigured.create(newKeySerde()); + final ImprovedKStream input = + builder.streamInput(ConfiguredConsumed.with(keySerde, valueSerde)); + input.toOutputTopic(ConfiguredProduced.with(keySerde, valueSerde)); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java index 58f6d7af..3418860f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,8 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.ImprovedKTable; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -33,8 +35,6 @@ import lombok.NoArgsConstructor; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Produced; @@ -43,15 +43,15 @@ public class WordCount implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream textLines = builder.streamInput(); + final ImprovedKStream textLines = builder.streamInput(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); - final KTable wordCounts = textLines + final ImprovedKTable wordCounts = textLines .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) .groupBy((key, word) -> word) .count(Materialized.as("counts")); - wordCounts.toStream().to(builder.getTopics().getOutputTopic(), Produced.valueSerde(Serdes.Long())); + wordCounts.toStream().toOutputTopic(Produced.valueSerde(Serdes.Long())); } @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java index 16b8f739..7d737b1c 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,6 +24,8 @@ package com.bakdata.kafka.test_applications; +import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.ImprovedKTable; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -34,8 +36,6 @@ import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.common.serialization.Serdes.StringSerde; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Produced; @@ -44,16 +44,16 @@ public class WordCountPattern implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final KStream textLines = builder.streamInputPattern(); + final ImprovedKStream textLines = builder.streamInputPattern(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); - final KTable wordCounts = textLines + final ImprovedKTable wordCounts = textLines .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) .groupBy((key, word) -> word) .count(Materialized.as("counts")); final Serde longValueSerde = Serdes.Long(); - wordCounts.toStream().to(builder.getTopics().getOutputTopic(), Produced.valueSerde(longValueSerde)); + wordCounts.toStream().toOutputTopic(Produced.valueSerde(longValueSerde)); } @Override From 59591d5ba57ab5c4eb5876420e8762ea85e7826c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 6 Jan 2025 15:59:52 +0100 Subject: [PATCH 02/72] Simplify writing to output topics --- .../java/com/bakdata/kafka/ConfiguredProduced.java | 8 ++++---- .../test_applications/MirrorWithNonDefaultSerde.java | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java index 7d350a00..49f212a9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java @@ -37,8 +37,8 @@ public class ConfiguredProduced { private final Preconfigured> keySerde; private final Preconfigured> valueSerde; - private final StreamPartitioner partitioner; - private final String processorName; + private final StreamPartitioner streamPartitioner; + private final String name; public static ConfiguredProduced keySerde(final Preconfigured> keySerde) { return with(keySerde, null); @@ -58,9 +58,9 @@ public static ConfiguredProduced as(final String processorName) { } Produced configure(final Configurator configurator) { - return Produced.as(this.processorName) + return Produced.as(this.name) .withKeySerde(configurator.configureForKeys(this.keySerde)) .withValueSerde(configurator.configureForValues(this.valueSerde)) - .withStreamPartitioner(this.partitioner); + .withStreamPartitioner(this.streamPartitioner); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java index 87a5d051..90f9de6f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java @@ -41,18 +41,18 @@ @NoArgsConstructor public class MirrorWithNonDefaultSerde implements StreamsApp { - public static Serde newKeySerde() { - return new SpecificAvroSerde<>(); + public static Preconfigured> newKeySerde() { + return Preconfigured.create(new SpecificAvroSerde<>()); } - public static Serde newValueSerde() { - return new SpecificAvroSerde<>(); + public static Preconfigured> newValueSerde() { + return Preconfigured.create(new SpecificAvroSerde<>()); } @Override public void buildTopology(final TopologyBuilder builder) { - final Preconfigured> valueSerde = Preconfigured.create(newValueSerde()); - final Preconfigured> keySerde = Preconfigured.create(newKeySerde()); + final Preconfigured> valueSerde = newValueSerde(); + final Preconfigured> keySerde = newKeySerde(); final ImprovedKStream input = builder.streamInput(ConfiguredConsumed.with(keySerde, valueSerde)); input.toOutputTopic(ConfiguredProduced.with(keySerde, valueSerde)); From 322eb93ffbcde2b2e2d2c6f313253a1d9bea1de2 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 07:41:38 +0100 Subject: [PATCH 03/72] Add error handling --- streams-bootstrap-core/build.gradle.kts | 1 + .../kafka/ImprovedBranchedKStream.java | 49 ++ .../kafka/ImprovedBranchedKStreamImpl.java | 78 +++ .../kafka/ImprovedCogroupedStreamImpl.java | 16 +- .../kafka/ImprovedKGroupedStreamImpl.java | 28 +- .../kafka/ImprovedKGroupedTableImpl.java | 22 +- .../com/bakdata/kafka/ImprovedKStream.java | 135 ++++- .../bakdata/kafka/ImprovedKStreamImpl.java | 516 +++++++++++++++--- .../com/bakdata/kafka/ImprovedKTableImpl.java | 102 ++-- ...vedSessionWindowedCogroupedStreamImpl.java | 8 +- .../ImprovedSessionWindowedStreamImpl.java | 26 +- ...provedTimeWindowedCogroupedStreamImpl.java | 8 +- .../kafka/ImprovedTimeWindowedStreamImpl.java | 26 +- .../java/com/bakdata/kafka/KErrorStream.java | 38 ++ .../bakdata/kafka/KeyValueKErrorStream.java | 54 ++ .../com/bakdata/kafka/StreamsContext.java | 23 +- .../com/bakdata/kafka/TopologyBuilder.java | 12 +- .../com/bakdata/kafka/ValueKErrorStream.java | 54 ++ 18 files changed, 974 insertions(+), 222 deletions(-) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index 27675109..4fbfdd84 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -19,6 +19,7 @@ dependencies { version = "2.0.9" ) // required because other dependencies use Slf4j 1.x which is not properly resolved if this library is used in test scope implementation(group = "org.jooq", name = "jool", version = "0.9.14") + implementation(group = "com.bakdata.kafka", name = "error-handling-core", version = "1.6.1-SNAPSHOT") val junitVersion: String by project testRuntimeOnly(group = "org.junit.jupiter", name = "junit-jupiter-engine", version = junitVersion) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java new file mode 100644 index 00000000..9c19e4bf --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java @@ -0,0 +1,49 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.Map; +import org.apache.kafka.streams.kstream.Branched; +import org.apache.kafka.streams.kstream.BranchedKStream; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Predicate; + +public interface ImprovedBranchedKStream extends BranchedKStream { + + @Override + ImprovedBranchedKStream branch(Predicate predicate); + + @Override + ImprovedBranchedKStream branch(Predicate predicate, Branched branched); + + @Override + Map> defaultBranch(); + + @Override + Map> defaultBranch(Branched branched); + + @Override + Map> noDefaultBranch(); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java new file mode 100644 index 00000000..bcd30805 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java @@ -0,0 +1,78 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.stream.Collectors; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.streams.kstream.Branched; +import org.apache.kafka.streams.kstream.BranchedKStream; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.Predicate; + +@RequiredArgsConstructor +public class ImprovedBranchedKStreamImpl implements ImprovedBranchedKStream { + + private final @NonNull BranchedKStream wrapped; + private final @NonNull StreamsContext context; + + @Override + public ImprovedBranchedKStream branch(final Predicate predicate) { + return this.context.wrap(this.wrapped.branch(predicate)); + } + + @Override + public ImprovedBranchedKStream branch(final Predicate predicate, + final Branched branched) { + return this.context.wrap(this.wrapped.branch(predicate, branched)); + } + + @Override + public Map> defaultBranch() { + return this.wrap(this.wrapped.defaultBranch()); + } + + @Override + public Map> defaultBranch(final Branched branched) { + return this.wrap(this.wrapped.defaultBranch(branched)); + } + + @Override + public Map> noDefaultBranch() { + return this.wrap(this.wrapped.noDefaultBranch()); + } + + private Map> wrap(final Map> streamMap) { + return streamMap.entrySet() + .stream() + .collect(Collectors.toMap(Entry::getKey, this::wrapValue)); + } + + private ImprovedKStream wrapValue(final Entry> entry) { + return this.context.wrap(entry.getValue()); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java index ebaddeca..3e5d9db7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java @@ -48,43 +48,43 @@ class ImprovedCogroupedStreamImpl implements ImprovedCogroupedKStream ImprovedCogroupedKStream cogroup(final KGroupedStream groupedStream, final Aggregator aggregator) { - return this.context.newCogroupedStream(this.wrapped.cogroup(groupedStream, aggregator)); + return this.context.wrap(this.wrapped.cogroup(groupedStream, aggregator)); } @Override public ImprovedKTable aggregate(final Initializer initializer) { - return this.context.newTable(this.wrapped.aggregate(initializer)); + return this.context.wrap(this.wrapped.aggregate(initializer)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Named named) { - return this.context.newTable(this.wrapped.aggregate(initializer, named)); + return this.context.wrap(this.wrapped.aggregate(initializer, named)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, materialized)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, named, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, named, materialized)); } @Override public ImprovedTimeWindowedCogroupedKStream windowedBy(final Windows windows) { - return this.context.newTimeWindowedCogroupedStream(this.wrapped.windowedBy(windows)); + return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override public ImprovedTimeWindowedCogroupedKStream windowedBy(final SlidingWindows windows) { - return this.context.newTimeWindowedCogroupedStream(this.wrapped.windowedBy(windows)); + return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override public ImprovedSessionWindowedCogroupedKStream windowedBy(final SessionWindows windows) { - return this.context.newSessionWindowedCogroupedStream(this.wrapped.windowedBy(windows)); + return this.context.wrap(this.wrapped.windowedBy(windows)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java index 4cd25946..743c06f9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -47,79 +47,79 @@ class ImprovedKGroupedStreamImpl implements ImprovedKGroupedStream { @Override public ImprovedKTable count() { - return this.context.newTable(this.wrapped.count()); + return this.context.wrap(this.wrapped.count()); } @Override public ImprovedKTable count(final Named named) { - return this.context.newTable(this.wrapped.count(named)); + return this.context.wrap(this.wrapped.count(named)); } @Override public ImprovedKTable count(final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(materialized)); + return this.context.wrap(this.wrapped.count(materialized)); } @Override public ImprovedKTable count(final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(named, materialized)); + return this.context.wrap(this.wrapped.count(named, materialized)); } @Override public ImprovedKTable reduce(final Reducer reducer) { - return this.context.newTable(this.wrapped.reduce(reducer)); + return this.context.wrap(this.wrapped.reduce(reducer)); } @Override public ImprovedKTable reduce(final Reducer reducer, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(reducer, materialized)); + return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } @Override public ImprovedKTable reduce(final Reducer reducer, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(reducer, named, materialized)); + return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, named, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, named, materialized)); } @Override public ImprovedTimeWindowedKStream windowedBy(final Windows windows) { - return this.context.newTimeWindowedStream(this.wrapped.windowedBy(windows)); + return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override public ImprovedTimeWindowedKStream windowedBy(final SlidingWindows windows) { - return this.context.newTimeWindowedStream(this.wrapped.windowedBy(windows)); + return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override public ImprovedSessionWindowedKStream windowedBy(final SessionWindows windows) { - return this.context.newSessionWindowedStream(this.wrapped.windowedBy(windows)); + return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override public ImprovedCogroupedKStream cogroup(final Aggregator aggregator) { - return this.context.newCogroupedStream(this.wrapped.cogroup(aggregator)); + return this.context.wrap(this.wrapped.cogroup(aggregator)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java index a1a264a7..a82984f6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java @@ -43,40 +43,40 @@ class ImprovedKGroupedTableImpl implements ImprovedKGroupedTable { @Override public ImprovedKTable count(final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(materialized)); + return this.context.wrap(this.wrapped.count(materialized)); } @Override public ImprovedKTable count(final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(named, materialized)); + return this.context.wrap(this.wrapped.count(named, materialized)); } @Override public ImprovedKTable count() { - return this.context.newTable(this.wrapped.count()); + return this.context.wrap(this.wrapped.count()); } @Override public ImprovedKTable count(final Named named) { - return this.context.newTable(this.wrapped.count(named)); + return this.context.wrap(this.wrapped.count(named)); } @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(adder, subtractor, materialized)); + return this.context.wrap(this.wrapped.reduce(adder, subtractor, materialized)); } @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(adder, subtractor, materialized)); + return this.context.wrap(this.wrapped.reduce(adder, subtractor, materialized)); } @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor) { - return this.context.newTable(this.wrapped.reduce(adder, subtractor)); + return this.context.wrap(this.wrapped.reduce(adder, subtractor)); } @Override @@ -84,7 +84,7 @@ public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); } @Override @@ -92,20 +92,20 @@ public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor) { - return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor)); + return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor)); } @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named) { - return this.context.newTable(this.wrapped.aggregate(initializer, adder, subtractor, named)); + return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor, named)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index be1a8c7f..0f8f47c9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -48,6 +48,7 @@ import org.apache.kafka.streams.kstream.ValueTransformerSupplier; import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; import org.apache.kafka.streams.state.KeyValueStore; public interface ImprovedKStream extends KStream { @@ -74,53 +75,147 @@ public interface ImprovedKStream extends KStream { ImprovedKStream map( KeyValueMapper> mapper); + KErrorStream mapCapturingErrors( + KeyValueMapper> mapper); + + KErrorStream mapCapturingErrors( + KeyValueMapper> mapper, + java.util.function.Predicate errorFilter); + @Override ImprovedKStream map( KeyValueMapper> mapper, Named named); + KErrorStream mapCapturingErrors( + KeyValueMapper> mapper, Named named); + + KErrorStream mapCapturingErrors( + KeyValueMapper> mapper, + java.util.function.Predicate errorFilter, Named named); + @Override ImprovedKStream mapValues(ValueMapper mapper); + KErrorStream mapValuesCapturingErrors(ValueMapper mapper); + + KErrorStream mapValuesCapturingErrors(ValueMapper mapper, + java.util.function.Predicate errorFilter); + @Override ImprovedKStream mapValues(ValueMapper mapper, Named named); + KErrorStream mapValuesCapturingErrors(ValueMapper mapper, Named named); + + KErrorStream mapValuesCapturingErrors(ValueMapper mapper, + java.util.function.Predicate errorFilter, Named named); + @Override ImprovedKStream mapValues(ValueMapperWithKey mapper); + KErrorStream mapValuesCapturingErrors( + ValueMapperWithKey mapper); + + KErrorStream mapValuesCapturingErrors( + ValueMapperWithKey mapper, + java.util.function.Predicate errorFilter); + @Override ImprovedKStream mapValues(ValueMapperWithKey mapper, Named named); + KErrorStream mapValuesCapturingErrors( + ValueMapperWithKey mapper, Named named); + + KErrorStream mapValuesCapturingErrors( + ValueMapperWithKey mapper, + java.util.function.Predicate errorFilter, Named named); + @Override ImprovedKStream flatMap( KeyValueMapper>> mapper); + KErrorStream flatMapCapturingErrors( + KeyValueMapper>> mapper); + + KErrorStream flatMapCapturingErrors( + KeyValueMapper>> mapper, + java.util.function.Predicate errorFilter); + @Override ImprovedKStream flatMap( KeyValueMapper>> mapper, Named named); + KErrorStream flatMapCapturingErrors( + KeyValueMapper>> mapper, + Named named); + + KErrorStream flatMapCapturingErrors( + KeyValueMapper>> mapper, + java.util.function.Predicate errorFilter, Named named); + @Override ImprovedKStream flatMapValues(ValueMapper> mapper); + KErrorStream flatMapValuesCapturingErrors( + ValueMapper> mapper); + + KErrorStream flatMapValuesCapturingErrors( + ValueMapper> mapper, + java.util.function.Predicate errorFilter); + @Override ImprovedKStream flatMapValues(ValueMapper> mapper, Named named); + KErrorStream flatMapValuesCapturingErrors( + ValueMapper> mapper, + Named named); + + KErrorStream flatMapValuesCapturingErrors( + ValueMapper> mapper, + java.util.function.Predicate errorFilter, Named named); + @Override ImprovedKStream flatMapValues( ValueMapperWithKey> mapper); + KErrorStream flatMapValuesCapturingErrors( + ValueMapperWithKey> mapper); + + KErrorStream flatMapValuesCapturingErrors( + ValueMapperWithKey> mapper, + java.util.function.Predicate errorFilter); + @Override ImprovedKStream flatMapValues( ValueMapperWithKey> mapper, Named named); + KErrorStream flatMapValuesCapturingErrors( + ValueMapperWithKey> mapper, Named named); + + KErrorStream flatMapValuesCapturingErrors( + ValueMapperWithKey> mapper, + java.util.function.Predicate errorFilter, Named named); + @Override ImprovedKStream peek(ForeachAction action); @Override ImprovedKStream peek(ForeachAction action, Named named); + @Override + ImprovedKStream[] branch(Named named, Predicate... predicates); + + @Override + ImprovedKStream[] branch(Predicate... predicates); + + @Override + ImprovedBranchedKStream split(); + + @Override + ImprovedBranchedKStream split(Named named); + @Override ImprovedKStream merge(KStream stream); @@ -371,12 +466,30 @@ ImprovedKStream flatTransformValues( @Override ImprovedKStream process( - org.apache.kafka.streams.processor.api.ProcessorSupplier processorSupplier, + ProcessorSupplier processorSupplier, + String... stateStoreNames); + + KErrorStream processCapturingErrors( + ProcessorSupplier processorSupplier, + String... stateStoreNames); + + KErrorStream processCapturingErrors( + ProcessorSupplier processorSupplier, + java.util.function.Predicate errorFilter, String... stateStoreNames); @Override ImprovedKStream process( - org.apache.kafka.streams.processor.api.ProcessorSupplier processorSupplier, + ProcessorSupplier processorSupplier, + Named named, String... stateStoreNames); + + KErrorStream processCapturingErrors( + ProcessorSupplier processorSupplier, + Named named, String... stateStoreNames); + + KErrorStream processCapturingErrors( + ProcessorSupplier processorSupplier, + java.util.function.Predicate errorFilter, Named named, String... stateStoreNames); @Override @@ -384,8 +497,26 @@ ImprovedKStream processValues( FixedKeyProcessorSupplier processorSupplier, String... stateStoreNames); + KErrorStream processValuesCapturingErrors( + FixedKeyProcessorSupplier processorSupplier, + String... stateStoreNames); + + KErrorStream processValuesCapturingErrors( + FixedKeyProcessorSupplier processorSupplier, + java.util.function.Predicate errorFilter, + String... stateStoreNames); + @Override ImprovedKStream processValues( FixedKeyProcessorSupplier processorSupplier, Named named, String... stateStoreNames); + + KErrorStream processValuesCapturingErrors( + FixedKeyProcessorSupplier processorSupplier, + Named named, String... stateStoreNames); + + KErrorStream processValuesCapturingErrors( + FixedKeyProcessorSupplier processorSupplier, + java.util.function.Predicate errorFilter, + Named named, String... stateStoreNames); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 33150b8f..af5b9388 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -24,11 +24,11 @@ package com.bakdata.kafka; +import java.util.Arrays; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.kstream.BranchedKStream; import org.apache.kafka.streams.kstream.ForeachAction; import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.Grouped; @@ -64,74 +64,167 @@ class ImprovedKStreamImpl implements ImprovedKStream { @Override public ImprovedKStream filter(final Predicate predicate) { - return this.context.newStream(this.wrapped.filter(predicate)); + return this.context.wrap(this.wrapped.filter(predicate)); } @Override public ImprovedKStream filter(final Predicate predicate, final Named named) { - return this.context.newStream(this.wrapped.filter(predicate, named)); + return this.context.wrap(this.wrapped.filter(predicate, named)); } @Override public ImprovedKStream filterNot(final Predicate predicate) { - return this.context.newStream(this.wrapped.filterNot(predicate)); + return this.context.wrap(this.wrapped.filterNot(predicate)); } @Override public ImprovedKStream filterNot(final Predicate predicate, final Named named) { - return this.context.newStream(this.wrapped.filterNot(predicate, named)); + return this.context.wrap(this.wrapped.filterNot(predicate, named)); } @Override public ImprovedKStream selectKey(final KeyValueMapper mapper) { - return this.context.newStream(this.wrapped.selectKey(mapper)); + return this.context.wrap(this.wrapped.selectKey(mapper)); } @Override public ImprovedKStream selectKey(final KeyValueMapper mapper, final Named named) { - return this.context.newStream(this.wrapped.selectKey(mapper, named)); + return this.context.wrap(this.wrapped.selectKey(mapper, named)); } @Override public ImprovedKStream map( final KeyValueMapper> mapper) { - return this.context.newStream(this.wrapped.map(mapper)); + return this.context.wrap(this.wrapped.map(mapper)); + } + + @Override + public KErrorStream mapCapturingErrors( + final KeyValueMapper> mapper) { + return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper)); + } + + @Override + public KErrorStream mapCapturingErrors( + final KeyValueMapper> mapper, + final java.util.function.Predicate errorFilter) { + return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper, errorFilter)); } @Override public ImprovedKStream map( final KeyValueMapper> mapper, final Named named) { - return this.context.newStream(this.wrapped.map(mapper, named)); + return this.context.wrap(this.wrapped.map(mapper, named)); + } + + @Override + public KErrorStream mapCapturingErrors( + final KeyValueMapper> mapper, + final Named named) { + return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper), named); + } + + @Override + public KErrorStream mapCapturingErrors( + final KeyValueMapper> mapper, + final java.util.function.Predicate errorFilter, final Named named) { + return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper, errorFilter), named); } @Override public ImprovedKStream mapValues(final ValueMapper mapper) { - return this.context.newStream(this.wrapped.mapValues(mapper)); + return this.context.wrap(this.wrapped.mapValues(mapper)); + } + + @Override + public KErrorStream mapValuesCapturingErrors( + final ValueMapper mapper) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper)); + } + + @Override + public KErrorStream mapValuesCapturingErrors(final ValueMapper mapper, + final java.util.function.Predicate errorFilter) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper, errorFilter)); } @Override public ImprovedKStream mapValues(final ValueMapper mapper, final Named named) { - return this.context.newStream(this.wrapped.mapValues(mapper, named)); + return this.context.wrap(this.wrapped.mapValues(mapper, named)); + } + + @Override + public KErrorStream mapValuesCapturingErrors(final ValueMapper mapper, + final Named named) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper), named); + } + + @Override + public KErrorStream mapValuesCapturingErrors(final ValueMapper mapper, + final java.util.function.Predicate errorFilter, final Named named) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper, errorFilter), named); } @Override public ImprovedKStream mapValues(final ValueMapperWithKey mapper) { - return this.context.newStream(this.wrapped.mapValues(mapper)); + return this.context.wrap(this.wrapped.mapValues(mapper)); + } + + @Override + public KErrorStream mapValuesCapturingErrors( + final ValueMapperWithKey mapper) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper)); + } + + @Override + public KErrorStream mapValuesCapturingErrors( + final ValueMapperWithKey mapper, + final java.util.function.Predicate errorFilter) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper, errorFilter)); } @Override public ImprovedKStream mapValues(final ValueMapperWithKey mapper, final Named named) { - return this.context.newStream(this.wrapped.mapValues(mapper, named)); + return this.context.wrap(this.wrapped.mapValues(mapper, named)); + } + + @Override + public KErrorStream mapValuesCapturingErrors( + final ValueMapperWithKey mapper, final Named named) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper), named); + } + + @Override + public KErrorStream mapValuesCapturingErrors( + final ValueMapperWithKey mapper, + final java.util.function.Predicate errorFilter, final Named named) { + return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper, errorFilter), + named); } @Override public ImprovedKStream flatMap( final KeyValueMapper>> mapper) { - return this.context.newStream(this.wrapped.flatMap(mapper)); + return this.context.wrap(this.wrapped.flatMap(mapper)); + } + + @Override + public KErrorStream flatMapCapturingErrors( + final KeyValueMapper>> mapper) { + return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper)); + } + + @Override + public KErrorStream flatMapCapturingErrors( + final KeyValueMapper>> mapper, + final java.util.function.Predicate errorFilter) { + return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper, errorFilter)); } @Override @@ -139,33 +232,106 @@ public ImprovedKStream flatMap( final KeyValueMapper>> mapper, final Named named) { - return this.context.newStream(this.wrapped.flatMap(mapper, named)); + return this.context.wrap(this.wrapped.flatMap(mapper, named)); + } + + @Override + public KErrorStream flatMapCapturingErrors( + final KeyValueMapper>> mapper, + final Named named) { + return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper), named); + } + + @Override + public KErrorStream flatMapCapturingErrors( + final KeyValueMapper>> mapper, + final java.util.function.Predicate errorFilter, final Named named) { + return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper, errorFilter), named); } @Override public ImprovedKStream flatMapValues( final ValueMapper> mapper) { - return this.context.newStream(this.wrapped.flatMapValues(mapper)); + return this.context.wrap(this.wrapped.flatMapValues(mapper)); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapper> mapper) { + return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper)); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapper> mapper, + final java.util.function.Predicate errorFilter) { + return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper, errorFilter)); } @Override public ImprovedKStream flatMapValues( final ValueMapper> mapper, final Named named) { - return this.context.newStream(this.wrapped.flatMapValues(mapper, named)); + return this.context.wrap(this.wrapped.flatMapValues(mapper, named)); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapper> mapper, final Named named) { + return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper), named); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapper> mapper, + final java.util.function.Predicate errorFilter, + final Named named) { + return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper, errorFilter), + named); } @Override public ImprovedKStream flatMapValues( final ValueMapperWithKey> mapper) { - return this.context.newStream(this.wrapped.flatMapValues(mapper)); + return this.context.wrap(this.wrapped.flatMapValues(mapper)); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapperWithKey> mapper) { + return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper)); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapperWithKey> mapper, + final java.util.function.Predicate errorFilter) { + return this.flatMapValuesCapturingErrors_( + ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper, errorFilter)); } @Override public ImprovedKStream flatMapValues( final ValueMapperWithKey> mapper, final Named named) { - return this.context.newStream(this.wrapped.flatMapValues(mapper, named)); + return this.context.wrap(this.wrapped.flatMapValues(mapper, named)); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapperWithKey> mapper, + final Named named) { + return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper), named); + } + + @Override + public KErrorStream flatMapValuesCapturingErrors( + final ValueMapperWithKey> mapper, + final java.util.function.Predicate errorFilter, final Named named) { + return this.flatMapValuesCapturingErrors_( + ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper, errorFilter), named); } @Override @@ -185,62 +351,66 @@ public void foreach(final ForeachAction action, final Name @Override public ImprovedKStream peek(final ForeachAction action) { - return this.context.newStream(this.wrapped.peek(action)); + return this.context.wrap(this.wrapped.peek(action)); } @Override public ImprovedKStream peek(final ForeachAction action, final Named named) { - return this.context.newStream(this.wrapped.peek(action, named)); + return this.context.wrap(this.wrapped.peek(action, named)); } @Override - public KStream[] branch(final Predicate... predicates) { - return this.wrapped.branch(predicates); + public ImprovedKStream[] branch(final Predicate... predicates) { + return Arrays.stream(this.wrapped.branch(predicates)) + .map(this.context::wrap) + .toArray(ImprovedKStream[]::new); } @Override - public KStream[] branch(final Named named, final Predicate... predicates) { - return this.wrapped.branch(named, predicates); + public ImprovedKStream[] branch(final Named named, final Predicate... predicates) { + return Arrays.stream(this.wrapped.branch(named, predicates)) + .map(this.context::wrap) + .toArray(ImprovedKStream[]::new); } @Override - public BranchedKStream split() { - return this.wrapped.split(); + public ImprovedBranchedKStream split() { + return this.context.wrap(this.wrapped.split()); } @Override - public BranchedKStream split(final Named named) { - return this.wrapped.split(named); + public ImprovedBranchedKStream split(final Named named) { + return this.context.wrap(this.wrapped.split(named)); } @Override public ImprovedKStream merge(final KStream stream) { - return this.context.newStream(this.wrapped.merge(stream)); + return this.context.wrap(this.wrapped.merge(stream)); } @Override public ImprovedKStream merge(final KStream stream, final Named named) { - return this.context.newStream(this.wrapped.merge(stream, named)); + return this.context.wrap(this.wrapped.merge(stream, named)); } @Override public ImprovedKStream through(final String topic) { - return this.context.newStream(this.wrapped.through(topic)); + return this.context.wrap(this.wrapped.through(topic)); } @Override public ImprovedKStream through(final String topic, final Produced produced) { - return this.context.newStream(this.wrapped.through(topic, produced)); + return this.context.wrap(this.wrapped.through(topic, produced)); } @Override public ImprovedKStream repartition() { - return this.context.newStream(this.wrapped.repartition()); + return this.context.wrap(this.wrapped.repartition()); } @Override public ImprovedKStream repartition(final Repartitioned repartitioned) { - return this.context.newStream(this.wrapped.repartition(repartitioned)); + return this.context.wrap(this.wrapped.repartition(repartitioned)); } @Override @@ -310,252 +480,252 @@ public void toErrorTopic(final ConfiguredProduced produced) { @Override public ImprovedKTable toTable() { - return this.context.newTable(this.wrapped.toTable()); + return this.context.wrap(this.wrapped.toTable()); } @Override public ImprovedKTable toTable(final Named named) { - return this.context.newTable(this.wrapped.toTable(named)); + return this.context.wrap(this.wrapped.toTable(named)); } @Override public ImprovedKTable toTable(final Materialized> materialized) { - return this.context.newTable(this.wrapped.toTable(materialized)); + return this.context.wrap(this.wrapped.toTable(materialized)); } @Override public ImprovedKTable toTable(final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.toTable(named, materialized)); + return this.context.wrap(this.wrapped.toTable(named, materialized)); } @Override public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector) { - return this.context.newGroupedStream(this.wrapped.groupBy(keySelector)); + return this.context.wrap(this.wrapped.groupBy(keySelector)); } @Override public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector, final Grouped grouped) { - return this.context.newGroupedStream(this.wrapped.groupBy(keySelector, grouped)); + return this.context.wrap(this.wrapped.groupBy(keySelector, grouped)); } @Override public ImprovedKGroupedStream groupByKey() { - return this.context.newGroupedStream(this.wrapped.groupByKey()); + return this.context.wrap(this.wrapped.groupByKey()); } @Override public ImprovedKGroupedStream groupByKey(final Grouped grouped) { - return this.context.newGroupedStream(this.wrapped.groupByKey(grouped)); + return this.context.wrap(this.wrapped.groupByKey(grouped)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { - return this.context.newStream(this.wrapped.join(otherStream, joiner, windows)); + return this.context.wrap(this.wrapped.join(otherStream, joiner, windows)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { - return this.context.newStream(this.wrapped.join(otherStream, joiner, windows)); + return this.context.wrap(this.wrapped.join(otherStream, joiner, windows)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.newStream(this.wrapped.join(otherStream, joiner, windows, streamJoined)); + return this.context.wrap(this.wrapped.join(otherStream, joiner, windows, streamJoined)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.newStream(this.wrapped.join(otherStream, joiner, windows, streamJoined)); + return this.context.wrap(this.wrapped.join(otherStream, joiner, windows, streamJoined)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { - return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows)); + return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { - return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows)); + return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); + return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.newStream(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); + return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { - return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows)); + return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { - return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows)); + return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); + return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.newStream(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); + return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); } @Override public ImprovedKStream join(final KTable table, final ValueJoiner joiner) { - return this.context.newStream(this.wrapped.join(table, joiner)); + return this.context.wrap(this.wrapped.join(table, joiner)); } @Override public ImprovedKStream join(final KTable table, final ValueJoinerWithKey joiner) { - return this.context.newStream(this.wrapped.join(table, joiner)); + return this.context.wrap(this.wrapped.join(table, joiner)); } @Override public ImprovedKStream join(final KTable table, final ValueJoiner joiner, final Joined joined) { - return this.context.newStream(this.wrapped.join(table, joiner, joined)); + return this.context.wrap(this.wrapped.join(table, joiner, joined)); } @Override public ImprovedKStream join(final KTable table, final ValueJoinerWithKey joiner, final Joined joined) { - return this.context.newStream(this.wrapped.join(table, joiner, joined)); + return this.context.wrap(this.wrapped.join(table, joiner, joined)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoiner joiner) { - return this.context.newStream(this.wrapped.leftJoin(table, joiner)); + return this.context.wrap(this.wrapped.leftJoin(table, joiner)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoinerWithKey joiner) { - return this.context.newStream(this.wrapped.leftJoin(table, joiner)); + return this.context.wrap(this.wrapped.leftJoin(table, joiner)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoiner joiner, final Joined joined) { - return this.context.newStream(this.wrapped.leftJoin(table, joiner, joined)); + return this.context.wrap(this.wrapped.leftJoin(table, joiner, joined)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoinerWithKey joiner, final Joined joined) { - return this.context.newStream(this.wrapped.leftJoin(table, joiner, joined)); + return this.context.wrap(this.wrapped.leftJoin(table, joiner, joined)); } @Override public ImprovedKStream join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner joiner) { - return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner)); + return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner)); } @Override public ImprovedKStream join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey joiner) { - return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner)); + return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner)); } @Override public ImprovedKStream join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner joiner, final Named named) { - return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner, named)); + return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner, named)); } @Override public ImprovedKStream join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey joiner, final Named named) { - return this.context.newStream(this.wrapped.join(globalTable, keySelector, joiner, named)); + return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner, named)); } @Override public ImprovedKStream leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner valueJoiner) { - return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); + return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); } @Override public ImprovedKStream leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey valueJoiner) { - return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); + return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); } @Override public ImprovedKStream leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner valueJoiner, final Named named) { - return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); + return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); } @Override public ImprovedKStream leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey valueJoiner, final Named named) { - return this.context.newStream(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); + return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); } @Override public ImprovedKStream transform( final TransformerSupplier> transformerSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.transform(transformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.transform(transformerSupplier, stateStoreNames)); } @Override public ImprovedKStream transform( final TransformerSupplier> transformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.transform(transformerSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.transform(transformerSupplier, named, stateStoreNames)); } @Override public ImprovedKStream flatTransform( final TransformerSupplier>> transformerSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.flatTransform(transformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, stateStoreNames)); } @Override @@ -563,28 +733,28 @@ public ImprovedKStream flatTransform( final TransformerSupplier>> transformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.flatTransform(transformerSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, named, stateStoreNames)); } @Override public ImprovedKStream transformValues( final ValueTransformerSupplier valueTransformerSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); } @Override public ImprovedKStream transformValues( final ValueTransformerSupplier valueTransformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); } @Override public ImprovedKStream transformValues( final ValueTransformerWithKeySupplier valueTransformerSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); } @Override @@ -592,21 +762,21 @@ public ImprovedKStream transformValues( final ValueTransformerWithKeySupplier valueTransformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); } @Override public ImprovedKStream flatTransformValues( final ValueTransformerSupplier> valueTransformerSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); } @Override public ImprovedKStream flatTransformValues( final ValueTransformerSupplier> valueTransformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream( + return this.context.wrap( this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); } @@ -614,7 +784,7 @@ public ImprovedKStream flatTransformValues( public ImprovedKStream flatTransformValues( final ValueTransformerWithKeySupplier> valueTransformerSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); } @Override @@ -622,7 +792,7 @@ public ImprovedKStream flatTransformValues( final ValueTransformerWithKeySupplier> valueTransformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream( + return this.context.wrap( this.wrapped.flatTransformValues(valueTransformerSupplier, named, stateStoreNames)); } @@ -644,27 +814,199 @@ public void process( public ImprovedKStream process( final ProcessorSupplier processorSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.process(processorSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.process(processorSupplier, stateStoreNames)); + } + + @Override + public KErrorStream processCapturingErrors( + final ProcessorSupplier processorSupplier, + final String... stateStoreNames) { + return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier), stateStoreNames); + } + + @Override + public KErrorStream processCapturingErrors( + final ProcessorSupplier processorSupplier, + final java.util.function.Predicate errorFilter, + final String... stateStoreNames) { + return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier, errorFilter), + stateStoreNames); } @Override public ImprovedKStream process( final ProcessorSupplier processorSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.process(processorSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.process(processorSupplier, named, stateStoreNames)); + } + + @Override + public KErrorStream processCapturingErrors( + final ProcessorSupplier processorSupplier, final Named named, + final String... stateStoreNames) { + return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier), named, + stateStoreNames); + } + + @Override + public KErrorStream processCapturingErrors( + final ProcessorSupplier processorSupplier, + final java.util.function.Predicate errorFilter, + final Named named, final String... stateStoreNames) { + return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier, errorFilter), + named, stateStoreNames); } @Override public ImprovedKStream processValues( final FixedKeyProcessorSupplier processorSupplier, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.processValues(processorSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.processValues(processorSupplier, stateStoreNames)); + } + + @Override + public KErrorStream processValuesCapturingErrors( + final FixedKeyProcessorSupplier processorSupplier, + final String... stateStoreNames) { + return this.processValuesCapturingErrors_(ErrorCapturingValueProcessor.captureErrors(processorSupplier), + stateStoreNames); + } + + @Override + public KErrorStream processValuesCapturingErrors( + final FixedKeyProcessorSupplier processorSupplier, + final java.util.function.Predicate errorFilter, final String... stateStoreNames) { + return this.processValuesCapturingErrors_( + ErrorCapturingValueProcessor.captureErrors(processorSupplier, errorFilter), + stateStoreNames + ); } @Override public ImprovedKStream processValues( final FixedKeyProcessorSupplier processorSupplier, final Named named, final String... stateStoreNames) { - return this.context.newStream(this.wrapped.processValues(processorSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.processValues(processorSupplier, named, stateStoreNames)); + } + + @Override + public KErrorStream processValuesCapturingErrors( + final FixedKeyProcessorSupplier processorSupplier, final Named named, + final String... stateStoreNames) { + return this.processValuesCapturingErrors_(ErrorCapturingValueProcessor.captureErrors(processorSupplier), named, + stateStoreNames); + } + + @Override + public KErrorStream processValuesCapturingErrors( + final FixedKeyProcessorSupplier processorSupplier, + final java.util.function.Predicate errorFilter, final Named named, + final String... stateStoreNames) { + return this.processValuesCapturingErrors_( + ErrorCapturingValueProcessor.captureErrors(processorSupplier, errorFilter), named, stateStoreNames); + } + + private KeyValueKErrorStream mapCapturingErrors_( + final KeyValueMapper>> mapper) { + final ImprovedKStream> map = this.map(mapper); + return new KeyValueKErrorStream<>(map); + } + + private KeyValueKErrorStream mapCapturingErrors_( + final KeyValueMapper>> mapper, final Named named) { + final ImprovedKStream> map = this.map(mapper, named); + return new KeyValueKErrorStream<>(map); + } + + private ValueKErrorStream mapValuesCapturingErrors_( + final ValueMapper> mapper) { + final ImprovedKStream> map = this.mapValues(mapper); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream mapValuesCapturingErrors_( + final ValueMapper> mapper, + final Named named) { + final ImprovedKStream> map = this.mapValues(mapper, named); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream mapValuesCapturingErrors_( + final ValueMapperWithKey> mapper) { + final ImprovedKStream> map = this.mapValues(mapper); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream mapValuesCapturingErrors_( + final ValueMapperWithKey> mapper, final Named named) { + final ImprovedKStream> map = this.mapValues(mapper, named); + return new ValueKErrorStream<>(map); + } + + private KeyValueKErrorStream flatMapCapturingErrors_( + final KeyValueMapper>>> mapper) { + final ImprovedKStream> map = this.flatMap(mapper); + return new KeyValueKErrorStream<>(map); + } + + private KeyValueKErrorStream flatMapCapturingErrors_( + final KeyValueMapper>>> mapper, final Named named) { + final ImprovedKStream> map = this.flatMap(mapper, named); + return new KeyValueKErrorStream<>(map); + } + + private ValueKErrorStream flatMapValuesCapturingErrors_( + final ValueMapper>> mapper) { + final ImprovedKStream> map = this.flatMapValues(mapper); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream flatMapValuesCapturingErrors_( + final ValueMapper>> mapper, final Named named) { + final ImprovedKStream> map = this.flatMapValues(mapper, named); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream flatMapValuesCapturingErrors_( + final ValueMapperWithKey>> mapper) { + final ImprovedKStream> map = this.flatMapValues(mapper); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream flatMapValuesCapturingErrors_( + final ValueMapperWithKey>> mapper, final Named named) { + final ImprovedKStream> map = this.flatMapValues(mapper, named); + return new ValueKErrorStream<>(map); + } + + private KeyValueKErrorStream processCapturingErrors_( + final ProcessorSupplier> processorSupplier, + final String... stateStoreNames) { + final ImprovedKStream> map = + this.process(processorSupplier, stateStoreNames); + return new KeyValueKErrorStream<>(map); + } + + private KeyValueKErrorStream processCapturingErrors_( + final ProcessorSupplier> processorSupplier, final Named named, + final String... stateStoreNames) { + final ImprovedKStream> map = + this.process(processorSupplier, named, stateStoreNames); + return new KeyValueKErrorStream<>(map); + } + + private ValueKErrorStream processValuesCapturingErrors_( + final FixedKeyProcessorSupplier> processorSupplier, + final String... stateStoreNames) { + final ImprovedKStream> map = this.processValues(processorSupplier, stateStoreNames); + return new ValueKErrorStream<>(map); + } + + private ValueKErrorStream processValuesCapturingErrors_( + final FixedKeyProcessorSupplier> processorSupplier, + final Named named, final String... stateStoreNames) { + final ImprovedKStream> map = + this.processValues(processorSupplier, named, stateStoreNames); + return new ValueKErrorStream<>(map); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java index ae170737..dff295f8 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -51,124 +51,124 @@ class ImprovedKTableImpl implements ImprovedKTable { @Override public ImprovedKTable filter(final Predicate predicate) { - return this.context.newTable(this.wrapped.filter(predicate)); + return this.context.wrap(this.wrapped.filter(predicate)); } @Override public ImprovedKTable filter(final Predicate predicate, final Named named) { - return this.context.newTable(this.wrapped.filter(predicate, named)); + return this.context.wrap(this.wrapped.filter(predicate, named)); } @Override public ImprovedKTable filter(final Predicate predicate, final Materialized> materialized) { - return this.context.newTable(this.wrapped.filter(predicate, materialized)); + return this.context.wrap(this.wrapped.filter(predicate, materialized)); } @Override public ImprovedKTable filter(final Predicate predicate, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.filter(predicate, named, materialized)); + return this.context.wrap(this.wrapped.filter(predicate, named, materialized)); } @Override public ImprovedKTable filterNot(final Predicate predicate) { - return this.context.newTable(this.wrapped.filterNot(predicate)); + return this.context.wrap(this.wrapped.filterNot(predicate)); } @Override public ImprovedKTable filterNot(final Predicate predicate, final Named named) { - return this.context.newTable(this.wrapped.filterNot(predicate, named)); + return this.context.wrap(this.wrapped.filterNot(predicate, named)); } @Override public ImprovedKTable filterNot(final Predicate predicate, final Materialized> materialized) { - return this.context.newTable(this.wrapped.filterNot(predicate, materialized)); + return this.context.wrap(this.wrapped.filterNot(predicate, materialized)); } @Override public ImprovedKTable filterNot(final Predicate predicate, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.filterNot(predicate, materialized)); + return this.context.wrap(this.wrapped.filterNot(predicate, materialized)); } @Override public ImprovedKTable mapValues(final ValueMapper mapper) { - return this.context.newTable(this.wrapped.mapValues(mapper)); + return this.context.wrap(this.wrapped.mapValues(mapper)); } @Override public ImprovedKTable mapValues(final ValueMapper mapper, final Named named) { - return this.context.newTable(this.wrapped.mapValues(mapper, named)); + return this.context.wrap(this.wrapped.mapValues(mapper, named)); } @Override public ImprovedKTable mapValues(final ValueMapperWithKey mapper) { - return this.context.newTable(this.wrapped.mapValues(mapper)); + return this.context.wrap(this.wrapped.mapValues(mapper)); } @Override public ImprovedKTable mapValues(final ValueMapperWithKey mapper, final Named named) { - return this.context.newTable(this.wrapped.mapValues(mapper, named)); + return this.context.wrap(this.wrapped.mapValues(mapper, named)); } @Override public ImprovedKTable mapValues(final ValueMapper mapper, final Materialized> materialized) { - return this.context.newTable(this.wrapped.mapValues(mapper, materialized)); + return this.context.wrap(this.wrapped.mapValues(mapper, materialized)); } @Override public ImprovedKTable mapValues(final ValueMapper mapper, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.mapValues(mapper, named, materialized)); + return this.context.wrap(this.wrapped.mapValues(mapper, named, materialized)); } @Override public ImprovedKTable mapValues(final ValueMapperWithKey mapper, final Materialized> materialized) { - return this.context.newTable(this.wrapped.mapValues(mapper, materialized)); + return this.context.wrap(this.wrapped.mapValues(mapper, materialized)); } @Override public ImprovedKTable mapValues(final ValueMapperWithKey mapper, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.mapValues(mapper, named, materialized)); + return this.context.wrap(this.wrapped.mapValues(mapper, named, materialized)); } @Override public ImprovedKStream toStream() { - return this.context.newStream(this.wrapped.toStream()); + return this.context.wrap(this.wrapped.toStream()); } @Override public ImprovedKStream toStream(final Named named) { - return this.context.newStream(this.wrapped.toStream(named)); + return this.context.wrap(this.wrapped.toStream(named)); } @Override public ImprovedKStream toStream(final KeyValueMapper mapper) { - return this.context.newStream(this.wrapped.toStream(mapper)); + return this.context.wrap(this.wrapped.toStream(mapper)); } @Override public ImprovedKStream toStream(final KeyValueMapper mapper, final Named named) { - return this.context.newStream(this.wrapped.toStream(mapper, named)); + return this.context.wrap(this.wrapped.toStream(mapper, named)); } @Override public ImprovedKTable suppress(final Suppressed suppressed) { - return this.context.newTable(this.wrapped.suppress(suppressed)); + return this.context.wrap(this.wrapped.suppress(suppressed)); } @Override public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final String... stateStoreNames) { - return this.context.newTable(this.wrapped.transformValues(transformerSupplier, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(transformerSupplier, stateStoreNames)); } @Override @@ -176,14 +176,14 @@ public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final Named named, final String... stateStoreNames) { - return this.context.newTable(this.wrapped.transformValues(transformerSupplier, named, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(transformerSupplier, named, stateStoreNames)); } @Override public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final Materialized> materialized, final String... stateStoreNames) { - return this.context.newTable(this.wrapped.transformValues(transformerSupplier, materialized, stateStoreNames)); + return this.context.wrap(this.wrapped.transformValues(transformerSupplier, materialized, stateStoreNames)); } @Override @@ -191,126 +191,126 @@ public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final Materialized> materialized, final Named named, final String... stateStoreNames) { - return this.context.newTable( + return this.context.wrap( this.wrapped.transformValues(transformerSupplier, materialized, named, stateStoreNames)); } @Override public ImprovedKGroupedTable groupBy( final KeyValueMapper> selector) { - return this.context.newGroupedTable(this.wrapped.groupBy(selector)); + return this.context.wrap(this.wrapped.groupBy(selector)); } @Override public ImprovedKGroupedTable groupBy( final KeyValueMapper> selector, final Grouped grouped) { - return this.context.newGroupedTable(this.wrapped.groupBy(selector, grouped)); + return this.context.wrap(this.wrapped.groupBy(selector, grouped)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner) { - return this.context.newTable(this.wrapped.join(other, joiner)); + return this.context.wrap(this.wrapped.join(other, joiner)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Named named) { - return this.context.newTable(this.wrapped.join(other, joiner, named)); + return this.context.wrap(this.wrapped.join(other, joiner, named)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.newTable(this.wrapped.join(other, joiner, materialized)); + return this.context.wrap(this.wrapped.join(other, joiner, materialized)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.join(other, joiner, materialized)); + return this.context.wrap(this.wrapped.join(other, joiner, materialized)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner) { - return this.context.newTable(this.wrapped.leftJoin(other, joiner)); + return this.context.wrap(this.wrapped.leftJoin(other, joiner)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Named named) { - return this.context.newTable(this.wrapped.leftJoin(other, joiner, named)); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, named)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.newTable(this.wrapped.leftJoin(other, joiner, materialized)); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, materialized)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.leftJoin(other, joiner, materialized)); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, materialized)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner) { - return this.context.newTable(this.wrapped.outerJoin(other, joiner)); + return this.context.wrap(this.wrapped.outerJoin(other, joiner)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Named named) { - return this.context.newTable(this.wrapped.outerJoin(other, joiner, named)); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, named)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.newTable(this.wrapped.outerJoin(other, joiner, materialized)); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, materialized)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.outerJoin(other, joiner, materialized)); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, materialized)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner) { - return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner)); + return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named) { - return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, named)); + return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, named)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined) { - return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined)); + return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, materialized)); + return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, materialized)); } @Override @@ -318,7 +318,7 @@ public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, named, materialized)); + return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, named, materialized)); } @Override @@ -326,35 +326,35 @@ public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { - return this.context.newTable(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined, materialized)); + return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined, materialized)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner) { - return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner)); + return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named) { - return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named)); + return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined) { - return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined)); + return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, materialized)); + return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, materialized)); } @Override @@ -362,7 +362,7 @@ public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named, materialized)); + return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named, materialized)); } @Override @@ -370,7 +370,7 @@ public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { - return this.context.newTable( + return this.context.wrap( this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, materialized)); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java index 7194695a..acad2a7b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java @@ -44,27 +44,27 @@ class ImprovedSessionWindowedCogroupedStreamImpl implements ImprovedSessio @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger) { - return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger)); + return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger)); } @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Named named) { - return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger, named)); + return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, named)); } @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, materialized)); } @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, sessionMerger, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, materialized)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java index e461fd00..57b03035 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java @@ -46,44 +46,44 @@ class ImprovedSessionWindowedStreamImpl implements ImprovedSessionWindowed @Override public ImprovedKTable, Long> count() { - return this.context.newTable(this.wrapped.count()); + return this.context.wrap(this.wrapped.count()); } @Override public ImprovedKTable, Long> count(final Named named) { - return this.context.newTable(this.wrapped.count(named)); + return this.context.wrap(this.wrapped.count(named)); } @Override public ImprovedKTable, Long> count( final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(materialized)); + return this.context.wrap(this.wrapped.count(materialized)); } @Override public ImprovedKTable, Long> count(final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(named, materialized)); + return this.context.wrap(this.wrapped.count(named, materialized)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Named named) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger, named)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, named)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); } @Override @@ -91,33 +91,33 @@ public ImprovedKTable, VR> aggregate(final Initializer init final Aggregator aggregator, final Merger sessionMerger, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer) { - return this.context.newTable(this.wrapped.reduce(reducer)); + return this.context.wrap(this.wrapped.reduce(reducer)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named) { - return this.context.newTable(this.wrapped.reduce(reducer, named)); + return this.context.wrap(this.wrapped.reduce(reducer, named)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(reducer, materialized)); + return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(reducer, named, materialized)); + return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } @Override public ImprovedSessionWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { - return this.context.newSessionWindowedStream(this.wrapped.emitStrategy(emitStrategy)); + return this.context.wrap(this.wrapped.emitStrategy(emitStrategy)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java index e8676636..a7239b1c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java @@ -42,23 +42,23 @@ class ImprovedTimeWindowedCogroupedStreamImpl implements ImprovedTimeWindo @Override public ImprovedKTable, V> aggregate(final Initializer initializer) { - return this.context.newTable(this.wrapped.aggregate(initializer)); + return this.context.wrap(this.wrapped.aggregate(initializer)); } @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named) { - return this.context.newTable(this.wrapped.aggregate(initializer, named)); + return this.context.wrap(this.wrapped.aggregate(initializer, named)); } @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, materialized)); } @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, named, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, named, materialized)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java index 6bf04f4d..46f1a36e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java @@ -45,76 +45,76 @@ class ImprovedTimeWindowedStreamImpl implements ImprovedTimeWindowedKStrea @Override public ImprovedKTable, Long> count() { - return this.context.newTable(this.wrapped.count()); + return this.context.wrap(this.wrapped.count()); } @Override public ImprovedKTable, Long> count(final Named named) { - return this.context.newTable(this.wrapped.count(named)); + return this.context.wrap(this.wrapped.count(named)); } @Override public ImprovedKTable, Long> count( final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(materialized)); + return this.context.wrap(this.wrapped.count(materialized)); } @Override public ImprovedKTable, Long> count(final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.count(named, materialized)); + return this.context.wrap(this.wrapped.count(named, materialized)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, named)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, named)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.aggregate(initializer, aggregator, materialized)); + return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer) { - return this.context.newTable(this.wrapped.reduce(reducer)); + return this.context.wrap(this.wrapped.reduce(reducer)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named) { - return this.context.newTable(this.wrapped.reduce(reducer, named)); + return this.context.wrap(this.wrapped.reduce(reducer, named)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(reducer, materialized)); + return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, final Materialized> materialized) { - return this.context.newTable(this.wrapped.reduce(reducer, named, materialized)); + return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } @Override public ImprovedTimeWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { - return this.context.newTimeWindowedStream(this.wrapped.emitStrategy(emitStrategy)); + return this.context.wrap(this.wrapped.emitStrategy(emitStrategy)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java new file mode 100644 index 00000000..e2997dca --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java @@ -0,0 +1,38 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.kstream.Named; + +public interface KErrorStream { + + ImprovedKStream values(); + + ImprovedKStream values(Named named); + + ImprovedKStream> errors(); + + ImprovedKStream> errors(Named named); +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java new file mode 100644 index 00000000..7b2e59fb --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.streams.kstream.Named; + +@RequiredArgsConstructor +class KeyValueKErrorStream implements KErrorStream { + private final @NonNull ImprovedKStream> stream; + + @Override + public ImprovedKStream values() { + return this.stream.flatMapValues(ProcessedKeyValue::getValues); + } + + @Override + public ImprovedKStream values(final Named named) { + return this.stream.flatMapValues(ProcessedKeyValue::getValues, named); + } + + @Override + public ImprovedKStream> errors() { + return this.stream.flatMap(ProcessedKeyValue::getErrors); + } + + @Override + public ImprovedKStream> errors(final Named named) { + return this.stream.flatMap(ProcessedKeyValue::getErrors, named); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index 9fbeab1e..d90a00e7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -28,6 +28,7 @@ import lombok.Getter; import lombok.NonNull; import lombok.Value; +import org.apache.kafka.streams.kstream.BranchedKStream; import org.apache.kafka.streams.kstream.CogroupedKStream; import org.apache.kafka.streams.kstream.KGroupedStream; import org.apache.kafka.streams.kstream.KGroupedTable; @@ -47,43 +48,47 @@ public class StreamsContext { @NonNull Configurator configurator; - public ImprovedKStream newStream(final KStream stream) { + public ImprovedKStream wrap(final KStream stream) { return new ImprovedKStreamImpl<>(stream, this); } - public ImprovedKGroupedStream newGroupedStream(final KGroupedStream stream) { + public ImprovedKGroupedStream wrap(final KGroupedStream stream) { return new ImprovedKGroupedStreamImpl<>(stream, this); } - public ImprovedTimeWindowedKStream newTimeWindowedStream( + public ImprovedTimeWindowedKStream wrap( final TimeWindowedKStream stream) { return new ImprovedTimeWindowedStreamImpl<>(stream, this); } - public ImprovedSessionWindowedKStream newSessionWindowedStream( + public ImprovedSessionWindowedKStream wrap( final SessionWindowedKStream stream) { return new ImprovedSessionWindowedStreamImpl<>(stream, this); } - public ImprovedTimeWindowedCogroupedKStream newTimeWindowedCogroupedStream( + public ImprovedTimeWindowedCogroupedKStream wrap( final TimeWindowedCogroupedKStream stream) { return new ImprovedTimeWindowedCogroupedStreamImpl<>(stream, this); } - public ImprovedSessionWindowedCogroupedKStream newSessionWindowedCogroupedStream( + public ImprovedSessionWindowedCogroupedKStream wrap( final SessionWindowedCogroupedKStream stream) { return new ImprovedSessionWindowedCogroupedStreamImpl<>(stream, this); } - public ImprovedCogroupedKStream newCogroupedStream(final CogroupedKStream stream) { + public ImprovedCogroupedKStream wrap(final CogroupedKStream stream) { return new ImprovedCogroupedStreamImpl<>(stream, this); } - public ImprovedKTable newTable(final KTable table) { + public ImprovedBranchedKStream wrap(final BranchedKStream stream) { + return new ImprovedBranchedKStreamImpl<>(stream, this); + } + + public ImprovedKTable wrap(final KTable table) { return new ImprovedKTableImpl<>(table, this); } - public ImprovedKGroupedTable newGroupedTable(final KGroupedTable table) { + public ImprovedKGroupedTable wrap(final KGroupedTable table) { return new ImprovedKGroupedTableImpl<>(table, this); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index bd050372..34f90eb7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -50,27 +50,27 @@ public class TopologyBuilder { Map kafkaProperties; public ImprovedKStream stream(final String topic) { - return this.getContext().newStream(this.streamsBuilder.stream(topic)); + return this.getContext().wrap(this.streamsBuilder.stream(topic)); } public ImprovedKStream stream(final String topic, final Consumed consumed) { - return this.getContext().newStream(this.streamsBuilder.stream(topic, consumed)); + return this.getContext().wrap(this.streamsBuilder.stream(topic, consumed)); } public ImprovedKStream stream(final Collection topics) { - return this.getContext().newStream(this.streamsBuilder.stream(topics)); + return this.getContext().wrap(this.streamsBuilder.stream(topics)); } public ImprovedKStream stream(final Collection topics, final Consumed consumed) { - return this.getContext().newStream(this.streamsBuilder.stream(topics, consumed)); + return this.getContext().wrap(this.streamsBuilder.stream(topics, consumed)); } public ImprovedKStream stream(final Pattern topicPattern) { - return this.getContext().newStream(this.streamsBuilder.stream(topicPattern)); + return this.getContext().wrap(this.streamsBuilder.stream(topicPattern)); } public ImprovedKStream stream(final Pattern topicPattern, final Consumed consumed) { - return this.getContext().newStream(this.streamsBuilder.stream(topicPattern, consumed)); + return this.getContext().wrap(this.streamsBuilder.stream(topicPattern, consumed)); } /** diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java new file mode 100644 index 00000000..eb6681ff --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java @@ -0,0 +1,54 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.streams.kstream.Named; + +@RequiredArgsConstructor +class ValueKErrorStream implements KErrorStream { + private final @NonNull ImprovedKStream> stream; + + @Override + public ImprovedKStream values() { + return this.stream.flatMapValues(ProcessedValue::getValues); + } + + @Override + public ImprovedKStream values(final Named named) { + return this.stream.flatMapValues(ProcessedValue::getValues, named); + } + + @Override + public ImprovedKStream> errors() { + return this.stream.flatMapValues(ProcessedValue::getErrors); + } + + @Override + public ImprovedKStream> errors(final Named named) { + return this.stream.flatMapValues(ProcessedValue::getErrors, named); + } +} From d321b8cd23c69f0b02e0b672685f714a9abb8e86 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 08:06:15 +0100 Subject: [PATCH 04/72] Add error handling --- .../java/com/bakdata/kafka/ConfiguredConsumed.java | 12 ++++++++++-- .../java/com/bakdata/kafka/ConfiguredProduced.java | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java index 12d894b6..ffda6052 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java @@ -61,9 +61,17 @@ public static ConfiguredConsumed as(final String processorName) { Consumed configure(final Configurator configurator) { return Consumed.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) + .withKeySerde(this.configureKeySerde(configurator)) + .withValueSerde(this.configureValueSerde(configurator)) .withOffsetResetPolicy(this.offsetResetPolicy) .withTimestampExtractor(this.timestampExtractor); } + + private Serde configureValueSerde(final Configurator configurator) { + return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); + } + + private Serde configureKeySerde(final Configurator configurator) { + return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); + } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java index 49f212a9..62d0d161 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java @@ -59,8 +59,16 @@ public static ConfiguredProduced as(final String processorName) { Produced configure(final Configurator configurator) { return Produced.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) + .withKeySerde(this.configureKeySerde(configurator)) + .withValueSerde(this.configuredValueSerde(configurator)) .withStreamPartitioner(this.streamPartitioner); } + + private Serde configuredValueSerde(final Configurator configurator) { + return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); + } + + private Serde configureKeySerde(final Configurator configurator) { + return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); + } } From 5377422e22a2c98e602880c8506c1a2062df7726 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 09:00:01 +0100 Subject: [PATCH 05/72] Add error handling --- .../kafka/ConfiguredRepartitioned.java | 92 +++++++++++++++++++ .../kafka/ImprovedCogroupedStreamImpl.java | 3 +- .../kafka/ImprovedKGroupedStreamImpl.java | 3 + .../com/bakdata/kafka/ImprovedKStream.java | 2 + .../bakdata/kafka/ImprovedKStreamImpl.java | 74 ++++++++++----- .../com/bakdata/kafka/ImprovedKTableImpl.java | 75 ++++++++++----- .../com/bakdata/kafka/StreamsContext.java | 54 +++++++++++ 7 files changed, 256 insertions(+), 47 deletions(-) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java new file mode 100644 index 00000000..b8738bf2 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java @@ -0,0 +1,92 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.processor.StreamPartitioner; + +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class ConfiguredRepartitioned { + + @With + private final Preconfigured> keySerde; + @With + private final Preconfigured> valueSerde; + @With + private final StreamPartitioner streamPartitioner; + @With + private final String name; + private final Integer numberOfPartitions; + + public static ConfiguredRepartitioned keySerde(final Preconfigured> keySerde) { + return with(keySerde, null); + } + + public static ConfiguredRepartitioned valueSerde(final Preconfigured> valueSerde) { + return with(null, valueSerde); + } + + public static ConfiguredRepartitioned with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ConfiguredRepartitioned<>(keySerde, valueSerde, null, null, null); + } + + public static ConfiguredRepartitioned as(final String name) { + return new ConfiguredRepartitioned<>(null, null, null, name, null); + } + + public static ConfiguredRepartitioned numberOfPartitions(final int numberOfPartitions) { + return new ConfiguredRepartitioned<>(null, null, null, null, numberOfPartitions); + } + + public static ConfiguredRepartitioned streamPartitioner(final StreamPartitioner partitioner) { + return new ConfiguredRepartitioned<>(null, null, partitioner, null, null); + } + + public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPartitions) { + return new ConfiguredRepartitioned<>(this.keySerde, this.valueSerde, this.streamPartitioner, this.name, + numberOfPartitions); + } + + Repartitioned configure(final Configurator configurator) { + return Repartitioned.as(this.name) + .withKeySerde(this.configureKeySerde(configurator)) + .withValueSerde(this.configuredValueSerde(configurator)) + .withStreamPartitioner(this.streamPartitioner) + .withNumberOfPartitions(this.numberOfPartitions); + } + + private Serde configuredValueSerde(final Configurator configurator) { + return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); + } + + private Serde configureKeySerde(final Configurator configurator) { + return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java index 3e5d9db7..4471783b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java @@ -48,7 +48,8 @@ class ImprovedCogroupedStreamImpl implements ImprovedCogroupedKStream ImprovedCogroupedKStream cogroup(final KGroupedStream groupedStream, final Aggregator aggregator) { - return this.context.wrap(this.wrapped.cogroup(groupedStream, aggregator)); + final KGroupedStream other = StreamsContext.maybeUnwrap(groupedStream); + return this.context.wrap(this.wrapped.cogroup(other, aggregator)); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java index 743c06f9..f9a8d01f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -24,6 +24,8 @@ package com.bakdata.kafka; +import lombok.AccessLevel; +import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.common.utils.Bytes; @@ -42,6 +44,7 @@ @RequiredArgsConstructor class ImprovedKGroupedStreamImpl implements ImprovedKGroupedStream { + @Getter(AccessLevel.PACKAGE) private final @NonNull KGroupedStream wrapped; private final @NonNull StreamsContext context; diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 0f8f47c9..971ed76f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -234,6 +234,8 @@ KErrorStream flatMapValuesCapturingErrors( @Override ImprovedKStream repartition(Repartitioned repartitioned); + ImprovedKStream repartition(ConfiguredRepartitioned repartitioned); + void toOutputTopic(); void toOutputTopic(Produced produced); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index af5b9388..2e42f444 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -25,6 +25,8 @@ package com.bakdata.kafka; import java.util.Arrays; +import lombok.AccessLevel; +import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.common.utils.Bytes; @@ -59,6 +61,7 @@ @RequiredArgsConstructor class ImprovedKStreamImpl implements ImprovedKStream { + @Getter(AccessLevel.PACKAGE) private final @NonNull KStream wrapped; private final @NonNull StreamsContext context; @@ -385,12 +388,14 @@ public ImprovedBranchedKStream split(final Named named) { @Override public ImprovedKStream merge(final KStream stream) { - return this.context.wrap(this.wrapped.merge(stream)); + final KStream other = StreamsContext.maybeUnwrap(stream); + return this.context.wrap(this.wrapped.merge(other)); } @Override public ImprovedKStream merge(final KStream stream, final Named named) { - return this.context.wrap(this.wrapped.merge(stream, named)); + final KStream other = StreamsContext.maybeUnwrap(stream); + return this.context.wrap(this.wrapped.merge(other, named)); } @Override @@ -413,6 +418,11 @@ public ImprovedKStream repartition(final Repartitioned repartitioned return this.context.wrap(this.wrapped.repartition(repartitioned)); } + @Override + public ImprovedKStream repartition(final ConfiguredRepartitioned repartitioned) { + return this.repartition(repartitioned.configure(this.context.getConfigurator())); + } + @Override public void to(final String topic) { this.wrapped.to(topic); @@ -523,132 +533,152 @@ public ImprovedKGroupedStream groupByKey(final Grouped grouped) { @Override public ImprovedKStream join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { - return this.context.wrap(this.wrapped.join(otherStream, joiner, windows)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.join(other, joiner, windows)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { - return this.context.wrap(this.wrapped.join(otherStream, joiner, windows)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.join(other, joiner, windows)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.wrap(this.wrapped.join(otherStream, joiner, windows, streamJoined)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.join(other, joiner, windows, streamJoined)); } @Override public ImprovedKStream join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.wrap(this.wrapped.join(otherStream, joiner, windows, streamJoined)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.join(other, joiner, windows, streamJoined)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { - return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { - return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows, streamJoined)); } @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.wrap(this.wrapped.leftJoin(otherStream, joiner, windows, streamJoined)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows, streamJoined)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { - return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { - return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows, streamJoined)); } @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { - return this.context.wrap(this.wrapped.outerJoin(otherStream, joiner, windows, streamJoined)); + final KStream other = StreamsContext.maybeUnwrap(otherStream); + return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows, streamJoined)); } @Override public ImprovedKStream join(final KTable table, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.join(table, joiner)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.join(other, joiner)); } @Override public ImprovedKStream join(final KTable table, final ValueJoinerWithKey joiner) { - return this.context.wrap(this.wrapped.join(table, joiner)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.join(other, joiner)); } @Override public ImprovedKStream join(final KTable table, final ValueJoiner joiner, final Joined joined) { - return this.context.wrap(this.wrapped.join(table, joiner, joined)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.join(other, joiner, joined)); } @Override public ImprovedKStream join(final KTable table, final ValueJoinerWithKey joiner, final Joined joined) { - return this.context.wrap(this.wrapped.join(table, joiner, joined)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.join(other, joiner, joined)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.leftJoin(table, joiner)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.leftJoin(other, joiner)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoinerWithKey joiner) { - return this.context.wrap(this.wrapped.leftJoin(table, joiner)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.leftJoin(other, joiner)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoiner joiner, final Joined joined) { - return this.context.wrap(this.wrapped.leftJoin(table, joiner, joined)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, joined)); } @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoinerWithKey joiner, final Joined joined) { - return this.context.wrap(this.wrapped.leftJoin(table, joiner, joined)); + final KTable other = StreamsContext.maybeUnwrap(table); + return this.context.wrap(this.wrapped.leftJoin(other, joiner, joined)); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java index dff295f8..931f1911 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -25,6 +25,8 @@ package com.bakdata.kafka; import java.util.function.Function; +import lombok.AccessLevel; +import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; import org.apache.kafka.common.utils.Bytes; @@ -46,6 +48,7 @@ @RequiredArgsConstructor class ImprovedKTableImpl implements ImprovedKTable { + @Getter(AccessLevel.PROTECTED) private final @NonNull KTable wrapped; private final @NonNull StreamsContext context; @@ -210,107 +213,123 @@ public ImprovedKGroupedTable groupBy( @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.join(other, joiner)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, joiner)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Named named) { - return this.context.wrap(this.wrapped.join(other, joiner, named)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, joiner, named)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.wrap(this.wrapped.join(other, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, joiner, materialized)); } @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.wrap(this.wrapped.join(other, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, joiner, materialized)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.leftJoin(other, joiner)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Named named) { - return this.context.wrap(this.wrapped.leftJoin(other, joiner, named)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner, named)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.wrap(this.wrapped.leftJoin(other, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner, materialized)); } @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.wrap(this.wrapped.leftJoin(other, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner, materialized)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.outerJoin(other, joiner)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Named named) { - return this.context.wrap(this.wrapped.outerJoin(other, joiner, named)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner, named)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.wrap(this.wrapped.outerJoin(other, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner, materialized)); } @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.wrap(this.wrapped.outerJoin(other, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner, materialized)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named) { - return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, named)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined) { - return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined)); } @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, materialized)); } @Override @@ -318,7 +337,8 @@ public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, named, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named, materialized)); } @Override @@ -326,35 +346,40 @@ public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { - return this.context.wrap(this.wrapped.join(other, foreignKeyExtractor, joiner, tableJoined, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner) { - return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named) { - return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined) { - return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined)); } @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Materialized> materialized) { - return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, materialized)); } @Override @@ -362,7 +387,8 @@ public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final Materialized> materialized) { - return this.context.wrap(this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, named, materialized)); + final KTable otherTable = StreamsContext.maybeUnwrap(other); + return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named, materialized)); } @Override @@ -370,8 +396,9 @@ public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { + final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap( - this.wrapped.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, materialized)); + this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index d90a00e7..91a74930 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -48,47 +48,101 @@ public class StreamsContext { @NonNull Configurator configurator; + static KStream maybeUnwrap(final KStream stream) { + if (stream instanceof ImprovedKStreamImpl) { + return ((ImprovedKStreamImpl) stream).getWrapped(); // Kafka Streams internally casts KStream to + // KStreamImpl in some cases + } + return stream; + } + + static KGroupedStream maybeUnwrap(final KGroupedStream stream) { + if (stream instanceof ImprovedKGroupedStream) { + return ((ImprovedKGroupedStreamImpl) stream).getWrapped(); // Kafka Streams internally casts + // KGroupedStream to KGroupedStreamImpl in some cases + } + return stream; + } + + static KTable maybeUnwrap(final KTable table) { + if (table instanceof ImprovedKTableImpl) { + return ((ImprovedKTableImpl) table).getWrapped(); // Kafka Streams internally casts KTable to + // KTableImpl in some cases + } + return table; + } + public ImprovedKStream wrap(final KStream stream) { + if (stream instanceof ImprovedKStream) { + return (ImprovedKStream) stream; + } return new ImprovedKStreamImpl<>(stream, this); } public ImprovedKGroupedStream wrap(final KGroupedStream stream) { + if (stream instanceof ImprovedKGroupedStream) { + return (ImprovedKGroupedStream) stream; + } return new ImprovedKGroupedStreamImpl<>(stream, this); } public ImprovedTimeWindowedKStream wrap( final TimeWindowedKStream stream) { + if (stream instanceof ImprovedTimeWindowedKStream) { + return (ImprovedTimeWindowedKStream) stream; + } return new ImprovedTimeWindowedStreamImpl<>(stream, this); } public ImprovedSessionWindowedKStream wrap( final SessionWindowedKStream stream) { + if (stream instanceof ImprovedSessionWindowedKStream) { + return (ImprovedSessionWindowedKStream) stream; + } return new ImprovedSessionWindowedStreamImpl<>(stream, this); } public ImprovedTimeWindowedCogroupedKStream wrap( final TimeWindowedCogroupedKStream stream) { + if (stream instanceof ImprovedTimeWindowedCogroupedKStream) { + return (ImprovedTimeWindowedCogroupedKStream) stream; + } return new ImprovedTimeWindowedCogroupedStreamImpl<>(stream, this); } public ImprovedSessionWindowedCogroupedKStream wrap( final SessionWindowedCogroupedKStream stream) { + if (stream instanceof ImprovedSessionWindowedCogroupedKStream) { + return (ImprovedSessionWindowedCogroupedKStream) stream; + } return new ImprovedSessionWindowedCogroupedStreamImpl<>(stream, this); } public ImprovedCogroupedKStream wrap(final CogroupedKStream stream) { + if (stream instanceof ImprovedCogroupedKStream) { + return (ImprovedCogroupedKStream) stream; + } return new ImprovedCogroupedStreamImpl<>(stream, this); } public ImprovedBranchedKStream wrap(final BranchedKStream stream) { + if (stream instanceof ImprovedBranchedKStream) { + return (ImprovedBranchedKStream) stream; + } return new ImprovedBranchedKStreamImpl<>(stream, this); } public ImprovedKTable wrap(final KTable table) { + if (table instanceof ImprovedKTable) { + return (ImprovedKTable) table; + } return new ImprovedKTableImpl<>(table, this); } public ImprovedKGroupedTable wrap(final KGroupedTable table) { + if (table instanceof ImprovedKGroupedTable) { + return (ImprovedKGroupedTable) table; + } return new ImprovedKGroupedTableImpl<>(table, this); } } From a5f272e7cf0e92c3269f1a8b0acd2fb94b3a9329 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 10:48:22 +0100 Subject: [PATCH 06/72] Add error handling --- .../bakdata/kafka/ConfiguredMaterialized.java | 109 +++++++++++++ .../kafka/ImprovedCogroupedKStream.java | 6 + .../kafka/ImprovedCogroupedStreamImpl.java | 12 ++ .../bakdata/kafka/ImprovedKGroupedStream.java | 17 ++ .../kafka/ImprovedKGroupedStreamImpl.java | 37 +++++ .../bakdata/kafka/ImprovedKGroupedTable.java | 19 +++ .../kafka/ImprovedKGroupedTableImpl.java | 37 +++++ .../com/bakdata/kafka/ImprovedKStream.java | 4 + .../bakdata/kafka/ImprovedKStreamImpl.java | 11 ++ .../com/bakdata/kafka/ImprovedKTable.java | 77 +++++++++ .../com/bakdata/kafka/ImprovedKTableImpl.java | 147 ++++++++++++++++++ ...provedSessionWindowedCogroupedKStream.java | 6 + ...vedSessionWindowedCogroupedStreamImpl.java | 12 ++ .../kafka/ImprovedSessionWindowedKStream.java | 21 +++ .../ImprovedSessionWindowedStreamImpl.java | 39 +++++ .../ImprovedTimeWindowedCogroupedKStream.java | 6 + ...provedTimeWindowedCogroupedStreamImpl.java | 12 ++ .../kafka/ImprovedTimeWindowedKStream.java | 19 +++ .../kafka/ImprovedTimeWindowedStreamImpl.java | 38 +++++ 19 files changed, 629 insertions(+) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java new file mode 100644 index 00000000..b7d24640 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java @@ -0,0 +1,109 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import lombok.AccessLevel; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.state.DslStoreSuppliers; + +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public class ConfiguredMaterialized { + + @With + private final Preconfigured> keySerde; + @With + private final Preconfigured> valueSerde; + private final String storeName; + @With + private final Duration retention; + @With + private final DslStoreSuppliers storeType; + private final Map topicConfig; + private final boolean loggingEnabled; + private final boolean cachingEnabled; + + public static ConfiguredMaterialized keySerde( + final Preconfigured> keySerde) { + return with(keySerde, null); + } + + public static ConfiguredMaterialized valueSerde( + final Preconfigured> valueSerde) { + return with(null, valueSerde); + } + + public static ConfiguredMaterialized with( + final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ConfiguredMaterialized<>(keySerde, valueSerde, null, null, null, new HashMap<>(), true, true); + } + + public static ConfiguredMaterialized as(final String storeName) { + return new ConfiguredMaterialized<>(null, null, storeName, null, null, new HashMap<>(), true, true); + } + + public static ConfiguredMaterialized as( + final DslStoreSuppliers storeSuppliers) { + return new ConfiguredMaterialized<>(null, null, null, null, storeSuppliers, new HashMap<>(), true, true); + } + + Materialized configure(final Configurator configurator) { + final Materialized materialized = Materialized.as(this.storeName) + .withKeySerde(this.configureKeySerde(configurator)) + .withValueSerde(this.configuredValueSerde(configurator)); + if (this.retention != null) { + materialized.withRetention(this.retention); + } + if (this.storeType != null) { + materialized.withStoreType(this.storeType); + } + if (this.loggingEnabled) { + materialized.withLoggingEnabled(this.topicConfig); + } else { + materialized.withLoggingDisabled(); + } + if (this.cachingEnabled) { + materialized.withCachingEnabled(); + } else { + materialized.withCachingDisabled(); + } + return materialized; + } + + private Serde configuredValueSerde(final Configurator configurator) { + return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); + } + + private Serde configureKeySerde(final Configurator configurator) { + return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java index 89a4e36e..4d36833d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -53,10 +53,16 @@ ImprovedCogroupedKStream cogroup(KGroupedStream groupedSt ImprovedKTable aggregate(Initializer initializer, Materialized> materialized); + ImprovedKTable aggregate(Initializer initializer, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable aggregate(Initializer initializer, Named named, Materialized> materialized); + ImprovedKTable aggregate(Initializer initializer, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedTimeWindowedCogroupedKStream windowedBy(Windows windows); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java index 4471783b..b2b8cc49 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java @@ -68,12 +68,24 @@ public ImprovedKTable aggregate(final Initializer initializer, return this.context.wrap(this.wrapped.aggregate(initializer, materialized)); } + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable aggregate(final Initializer initializer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, named, materialized)); } + @Override + public ImprovedKTable aggregate(final Initializer initializer, final Named named, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedTimeWindowedCogroupedKStream windowedBy(final Windows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index 3fc76ed8..8c54bae1 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -48,19 +48,30 @@ public interface ImprovedKGroupedStream extends KGroupedStream { @Override ImprovedKTable count(Materialized> materialized); + ImprovedKTable count(ConfiguredMaterialized> materialized); + @Override ImprovedKTable count(Named named, Materialized> materialized); + ImprovedKTable count(Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable reduce(Reducer reducer); @Override ImprovedKTable reduce(Reducer reducer, Materialized> materialized); + ImprovedKTable reduce(Reducer reducer, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable reduce(Reducer reducer, Named named, Materialized> materialized); + ImprovedKTable reduce(Reducer reducer, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator); @@ -68,10 +79,16 @@ ImprovedKTable reduce(Reducer reducer, Named named, ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, Materialized> materialized); + ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, Named named, Materialized> materialized); + ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + Named named, ConfiguredMaterialized> materialized); + @Override ImprovedTimeWindowedKStream windowedBy(Windows windows); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java index f9a8d01f..27014dc6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -63,12 +63,23 @@ public ImprovedKTable count(final Materialized count(final ConfiguredMaterialized> materialized) { + return this.count(materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } + @Override + public ImprovedKTable count(final Named named, + final ConfiguredMaterialized> materialized) { + return this.count(named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable reduce(final Reducer reducer) { return this.context.wrap(this.wrapped.reduce(reducer)); @@ -80,12 +91,24 @@ public ImprovedKTable reduce(final Reducer reducer, return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } + @Override + public ImprovedKTable reduce(final Reducer reducer, + final ConfiguredMaterialized> materialized) { + return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable reduce(final Reducer reducer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } + @Override + public ImprovedKTable reduce(final Reducer reducer, final Named named, + final ConfiguredMaterialized> materialized) { + return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator) { @@ -99,6 +122,13 @@ public ImprovedKTable aggregate(final Initializer initializer, return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator aggregator, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, @@ -106,6 +136,13 @@ public ImprovedKTable aggregate(final Initializer initializer, return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, named, materialized)); } + @Override + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator aggregator, final Named named, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedTimeWindowedKStream windowedBy(final Windows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java index e4d8e23a..65aecfe2 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -38,9 +38,14 @@ public interface ImprovedKGroupedTable extends KGroupedTable { @Override ImprovedKTable count(Materialized> materialized); + ImprovedKTable count(ConfiguredMaterialized> materialized); + @Override ImprovedKTable count(Named named, Materialized> materialized); + ImprovedKTable count(Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable count(); @@ -51,10 +56,16 @@ public interface ImprovedKGroupedTable extends KGroupedTable { ImprovedKTable reduce(Reducer adder, Reducer subtractor, Materialized> materialized); + ImprovedKTable reduce(Reducer adder, Reducer subtractor, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, Materialized> materialized); + ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable reduce(Reducer adder, Reducer subtractor); @@ -63,11 +74,19 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator subtractor, Materialized> materialized); + ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + Aggregator subtractor, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named, Materialized> materialized); + ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + Aggregator subtractor, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java index a82984f6..76089fe6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java @@ -46,12 +46,23 @@ public ImprovedKTable count(final Materialized count(final ConfiguredMaterialized> materialized) { + return this.count(materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } + @Override + public ImprovedKTable count(final Named named, + final ConfiguredMaterialized> materialized) { + return this.count(named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable count() { return this.context.wrap(this.wrapped.count()); @@ -68,12 +79,24 @@ public ImprovedKTable reduce(final Reducer adder, final Reducer subt return this.context.wrap(this.wrapped.reduce(adder, subtractor, materialized)); } + @Override + public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, + final ConfiguredMaterialized> materialized) { + return this.reduce(adder, subtractor, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(adder, subtractor, materialized)); } + @Override + public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, + final ConfiguredMaterialized> materialized) { + return this.reduce(adder, subtractor, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor) { return this.context.wrap(this.wrapped.reduce(adder, subtractor)); @@ -87,6 +110,13 @@ public ImprovedKTable aggregate(final Initializer initializer, return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); } + @Override + public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, + final Aggregator subtractor, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, adder, subtractor, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, @@ -95,6 +125,13 @@ public ImprovedKTable aggregate(final Initializer initializer, return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor, materialized)); } + @Override + public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, + final Aggregator subtractor, final Named named, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, adder, subtractor, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 971ed76f..c9bdb0c0 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -263,9 +263,13 @@ KErrorStream flatMapValuesCapturingErrors( @Override ImprovedKTable toTable(Materialized> materialized); + ImprovedKTable toTable(ConfiguredMaterialized> materialized); + @Override ImprovedKTable toTable(Named named, Materialized> materialized); + ImprovedKTable toTable(Named named, ConfiguredMaterialized> materialized); + @Override ImprovedKGroupedStream groupBy(KeyValueMapper keySelector); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 2e42f444..50bc2750 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -503,12 +503,23 @@ public ImprovedKTable toTable(final Materialized toTable(final ConfiguredMaterialized> materialized) { + return this.toTable(materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable toTable(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.toTable(named, materialized)); } + @Override + public ImprovedKTable toTable(final Named named, + final ConfiguredMaterialized> materialized) { + return this.toTable(named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector) { return this.context.wrap(this.wrapped.groupBy(keySelector)); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index 459e1326..583da8b4 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -53,10 +53,16 @@ public interface ImprovedKTable extends KTable { ImprovedKTable filter(Predicate predicate, Materialized> materialized); + ImprovedKTable filter(Predicate predicate, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable filter(Predicate predicate, Named named, Materialized> materialized); + ImprovedKTable filter(Predicate predicate, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable filterNot(Predicate predicate); @@ -67,10 +73,16 @@ ImprovedKTable filter(Predicate predicate, Named nam ImprovedKTable filterNot(Predicate predicate, Materialized> materialized); + ImprovedKTable filterNot(Predicate predicate, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable filterNot(Predicate predicate, Named named, Materialized> materialized); + ImprovedKTable filterNot(Predicate predicate, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable mapValues(ValueMapper mapper); @@ -87,18 +99,30 @@ ImprovedKTable filterNot(Predicate predicate, Named ImprovedKTable mapValues(ValueMapper mapper, Materialized> materialized); + ImprovedKTable mapValues(ValueMapper mapper, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable mapValues(ValueMapper mapper, Named named, Materialized> materialized); + ImprovedKTable mapValues(ValueMapper mapper, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable mapValues(ValueMapperWithKey mapper, Materialized> materialized); + ImprovedKTable mapValues(ValueMapperWithKey mapper, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, Materialized> materialized); + ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKStream toStream(); @@ -129,11 +153,20 @@ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, Materialized> materialized, String... stateStoreNames); + ImprovedKTable transformValues( + ValueTransformerWithKeySupplier transformerSupplier, + ConfiguredMaterialized> materialized, String... stateStoreNames); + @Override ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, Materialized> materialized, Named named, String... stateStoreNames); + ImprovedKTable transformValues( + ValueTransformerWithKeySupplier transformerSupplier, + ConfiguredMaterialized> materialized, Named named, + String... stateStoreNames); + @Override ImprovedKGroupedTable groupBy(KeyValueMapper> selector); @@ -152,10 +185,16 @@ ImprovedKTable join(KTable other, ValueJoiner ImprovedKTable join(KTable other, ValueJoiner joiner, Materialized> materialized); + ImprovedKTable join(KTable other, ValueJoiner joiner, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable join(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); + ImprovedKTable join(KTable other, ValueJoiner joiner, + Named named, ConfiguredMaterialized> materialized); + @Override ImprovedKTable leftJoin(KTable other, ValueJoiner joiner); @@ -170,11 +209,19 @@ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, Materialized> materialized); + ImprovedKTable leftJoin(KTable other, + ValueJoiner joiner, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); + ImprovedKTable leftJoin(KTable other, + ValueJoiner joiner, + Named named, ConfiguredMaterialized> materialized); + @Override ImprovedKTable outerJoin(KTable other, ValueJoiner joiner); @@ -189,11 +236,19 @@ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, Materialized> materialized); + ImprovedKTable outerJoin(KTable other, + ValueJoiner joiner, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); + ImprovedKTable outerJoin(KTable other, + ValueJoiner joiner, + Named named, ConfiguredMaterialized> materialized); + @Override ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); @@ -210,15 +265,26 @@ ImprovedKTable join(KTable other, Function fo ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Materialized> materialized); + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, ConfiguredMaterialized> materialized); + @Override ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, Materialized> materialized); + ImprovedKTable join(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); @@ -235,12 +301,23 @@ ImprovedKTable leftJoin(KTable other, Function ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Materialized> materialized); + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, ConfiguredMaterialized> materialized); + @Override ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, Materialized> materialized); + + ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + ValueJoiner joiner, TableJoined tableJoined, + ConfiguredMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java index 931f1911..f7f21249 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -68,12 +68,24 @@ public ImprovedKTable filter(final Predicate predica return this.context.wrap(this.wrapped.filter(predicate, materialized)); } + @Override + public ImprovedKTable filter(final Predicate predicate, + final ConfiguredMaterialized> materialized) { + return this.filter(predicate, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable filter(final Predicate predicate, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.filter(predicate, named, materialized)); } + @Override + public ImprovedKTable filter(final Predicate predicate, final Named named, + final ConfiguredMaterialized> materialized) { + return this.filter(predicate, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable filterNot(final Predicate predicate) { return this.context.wrap(this.wrapped.filterNot(predicate)); @@ -90,12 +102,24 @@ public ImprovedKTable filterNot(final Predicate pred return this.context.wrap(this.wrapped.filterNot(predicate, materialized)); } + @Override + public ImprovedKTable filterNot(final Predicate predicate, + final ConfiguredMaterialized> materialized) { + return this.filterNot(predicate, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable filterNot(final Predicate predicate, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.filterNot(predicate, materialized)); } + @Override + public ImprovedKTable filterNot(final Predicate predicate, final Named named, + final ConfiguredMaterialized> materialized) { + return this.filterNot(predicate, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable mapValues(final ValueMapper mapper) { return this.context.wrap(this.wrapped.mapValues(mapper)); @@ -123,24 +147,48 @@ public ImprovedKTable mapValues(final ValueMapper ImprovedKTable mapValues(final ValueMapper mapper, + final ConfiguredMaterialized> materialized) { + return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable mapValues(final ValueMapper mapper, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, named, materialized)); } + @Override + public ImprovedKTable mapValues(final ValueMapper mapper, final Named named, + final ConfiguredMaterialized> materialized) { + return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable mapValues(final ValueMapperWithKey mapper, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, materialized)); } + @Override + public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + final ConfiguredMaterialized> materialized) { + return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable mapValues(final ValueMapperWithKey mapper, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, named, materialized)); } + @Override + public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + final Named named, final ConfiguredMaterialized> materialized) { + return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream toStream() { return this.context.wrap(this.wrapped.toStream()); @@ -189,6 +237,13 @@ public ImprovedKTable transformValues( return this.context.wrap(this.wrapped.transformValues(transformerSupplier, materialized, stateStoreNames)); } + @Override + public ImprovedKTable transformValues( + final ValueTransformerWithKeySupplier transformerSupplier, + final ConfiguredMaterialized> materialized, final String... stateStoreNames) { + return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), stateStoreNames); + } + @Override public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, @@ -198,6 +253,15 @@ public ImprovedKTable transformValues( this.wrapped.transformValues(transformerSupplier, materialized, named, stateStoreNames)); } + @Override + public ImprovedKTable transformValues( + final ValueTransformerWithKeySupplier transformerSupplier, + final ConfiguredMaterialized> materialized, final Named named, + final String... stateStoreNames) { + return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), named, + stateStoreNames); + } + @Override public ImprovedKGroupedTable groupBy( final KeyValueMapper> selector) { @@ -232,6 +296,13 @@ public ImprovedKTable join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, joiner, materialized)); } + @Override + public ImprovedKTable join(final KTable other, + final ValueJoiner joiner, + final ConfiguredMaterialized> materialized) { + return this.join(other, joiner, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Named named, @@ -240,6 +311,13 @@ public ImprovedKTable join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, joiner, materialized)); } + @Override + public ImprovedKTable join(final KTable other, + final ValueJoiner joiner, final Named named, + final ConfiguredMaterialized> materialized) { + return this.join(other, joiner, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner) { @@ -262,6 +340,13 @@ public ImprovedKTable leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner, materialized)); } + @Override + public ImprovedKTable leftJoin(final KTable other, + final ValueJoiner joiner, + final ConfiguredMaterialized> materialized) { + return this.leftJoin(other, joiner, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Named named, @@ -270,6 +355,13 @@ public ImprovedKTable leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner, materialized)); } + @Override + public ImprovedKTable leftJoin(final KTable other, + final ValueJoiner joiner, final Named named, + final ConfiguredMaterialized> materialized) { + return this.leftJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner) { @@ -292,6 +384,13 @@ public ImprovedKTable outerJoin(final KTable other, return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner, materialized)); } + @Override + public ImprovedKTable outerJoin(final KTable other, + final ValueJoiner joiner, + final ConfiguredMaterialized> materialized) { + return this.outerJoin(other, joiner, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Named named, @@ -300,6 +399,13 @@ public ImprovedKTable outerJoin(final KTable other, return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner, materialized)); } + @Override + public ImprovedKTable outerJoin(final KTable other, + final ValueJoiner joiner, final Named named, + final ConfiguredMaterialized> materialized) { + return this.outerJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, @@ -332,6 +438,12 @@ public ImprovedKTable join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, materialized)); } + @Override + public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final ConfiguredMaterialized> materialized) { + return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, @@ -341,6 +453,13 @@ public ImprovedKTable join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named, materialized)); } + @Override + public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final Named named, + final ConfiguredMaterialized> materialized) { + return this.join(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, @@ -350,6 +469,13 @@ public ImprovedKTable join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } + @Override + public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final ConfiguredMaterialized> materialized) { + return this.join(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, @@ -382,6 +508,12 @@ public ImprovedKTable leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, materialized)); } + @Override + public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final ConfiguredMaterialized> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, @@ -391,6 +523,13 @@ public ImprovedKTable leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named, materialized)); } + @Override + public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final Named named, + final ConfiguredMaterialized> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, @@ -401,6 +540,14 @@ public ImprovedKTable leftJoin(final KTable other, this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, tableJoined, materialized)); } + @Override + public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, + final ValueJoiner joiner, final TableJoined tableJoined, + final ConfiguredMaterialized> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, + materialized.configure(this.context.getConfigurator())); + } + @Override public String queryableStoreName() { return this.wrapped.queryableStoreName(); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java index 19a4c199..ea5002fa 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -46,7 +46,13 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, Materialized> materialized); + ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, Named named, Materialized> materialized); + + ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + Named named, ConfiguredMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java index acad2a7b..ec73b63b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java @@ -61,10 +61,22 @@ public ImprovedKTable, V> aggregate(final Initializer initializer return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, materialized)); } + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, sessionMerger, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, materialized)); } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, + final Named named, final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, sessionMerger, named, materialized.configure(this.context.getConfigurator())); + } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java index 83ed6b08..896ffffe 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -47,10 +47,15 @@ public interface ImprovedSessionWindowedKStream extends SessionWindowedKSt @Override ImprovedKTable, Long> count(Materialized> materialized); + ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); + @Override ImprovedKTable, Long> count(Named named, Materialized> materialized); + ImprovedKTable, Long> count(Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, @@ -66,12 +71,22 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Materialized> materialized); + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Merger sessionMerger, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named, Materialized> materialized); + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Merger sessionMerger, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, V> reduce(Reducer reducer); @@ -82,10 +97,16 @@ ImprovedKTable, VR> aggregate(Initializer initializer, ImprovedKTable, V> reduce(Reducer reducer, Materialized> materialized); + ImprovedKTable, V> reduce(Reducer reducer, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, V> reduce(Reducer reducer, Named named, Materialized> materialized); + ImprovedKTable, V> reduce(Reducer reducer, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedSessionWindowedKStream emitStrategy(EmitStrategy emitStrategy); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java index 57b03035..b330d075 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java @@ -60,12 +60,24 @@ public ImprovedKTable, Long> count( return this.context.wrap(this.wrapped.count(materialized)); } + @Override + public ImprovedKTable, Long> count( + final ConfiguredMaterialized> materialized) { + return this.count(materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, Long> count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } + @Override + public ImprovedKTable, Long> count(final Named named, + final ConfiguredMaterialized> materialized) { + return this.count(named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger) { @@ -86,6 +98,13 @@ public ImprovedKTable, VR> aggregate(final Initializer init return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); } + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Merger sessionMerger, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, aggregator, sessionMerger, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, @@ -94,6 +113,14 @@ public ImprovedKTable, VR> aggregate(final Initializer init return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); } + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Merger sessionMerger, final Named named, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, aggregator, sessionMerger, named, + materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, V> reduce(final Reducer reducer) { return this.context.wrap(this.wrapped.reduce(reducer)); @@ -110,12 +137,24 @@ public ImprovedKTable, V> reduce(final Reducer reducer, return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, + final ConfiguredMaterialized> materialized) { + return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + final ConfiguredMaterialized> materialized) { + return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedSessionWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { return this.context.wrap(this.wrapped.emitStrategy(emitStrategy)); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java index 89a12ae1..cbd90455 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -44,7 +44,13 @@ public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindo ImprovedKTable, VOut> aggregate(Initializer initializer, Materialized> materialized); + ImprovedKTable, VOut> aggregate(Initializer initializer, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, Materialized> materialized); + + ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, + ConfiguredMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java index a7239b1c..752366b5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java @@ -56,9 +56,21 @@ public ImprovedKTable, V> aggregate(final Initializer initializer return this.context.wrap(this.wrapped.aggregate(initializer, materialized)); } + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, named, materialized)); } + + @Override + public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); + } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java index 8e60053b..e57ba7cc 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -46,10 +46,15 @@ public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream, Long> count(Materialized> materialized); + ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); + @Override ImprovedKTable, Long> count(Named named, Materialized> materialized); + ImprovedKTable, Long> count(Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator); @@ -64,11 +69,19 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Materialized> materialized); + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Named named, Materialized> materialized); + ImprovedKTable, VR> aggregate(Initializer initializer, + Aggregator aggregator, + Named named, ConfiguredMaterialized> materialized); + @Override ImprovedKTable, V> reduce(Reducer reducer); @@ -79,10 +92,16 @@ ImprovedKTable, VR> aggregate(Initializer initializer, ImprovedKTable, V> reduce(Reducer reducer, Materialized> materialized); + ImprovedKTable, V> reduce(Reducer reducer, + ConfiguredMaterialized> materialized); + @Override ImprovedKTable, V> reduce(Reducer reducer, Named named, Materialized> materialized); + ImprovedKTable, V> reduce(Reducer reducer, Named named, + ConfiguredMaterialized> materialized); + @Override ImprovedTimeWindowedKStream emitStrategy(EmitStrategy emitStrategy); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java index 46f1a36e..0c858628 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java @@ -59,12 +59,24 @@ public ImprovedKTable, Long> count( return this.context.wrap(this.wrapped.count(materialized)); } + @Override + public ImprovedKTable, Long> count( + final ConfiguredMaterialized> materialized) { + return this.count(materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, Long> count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } + @Override + public ImprovedKTable, Long> count(final Named named, + final ConfiguredMaterialized> materialized) { + return this.count(named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator) { @@ -84,6 +96,13 @@ public ImprovedKTable, VR> aggregate(final Initializer init return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, @@ -91,6 +110,13 @@ public ImprovedKTable, VR> aggregate(final Initializer init return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } + @Override + public ImprovedKTable, VR> aggregate(final Initializer initializer, + final Aggregator aggregator, final Named named, + final ConfiguredMaterialized> materialized) { + return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, V> reduce(final Reducer reducer) { return this.context.wrap(this.wrapped.reduce(reducer)); @@ -107,12 +133,24 @@ public ImprovedKTable, V> reduce(final Reducer reducer, return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, + final ConfiguredMaterialized> materialized) { + return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } + @Override + public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + final ConfiguredMaterialized> materialized) { + return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); + } + @Override public ImprovedTimeWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { return this.context.wrap(this.wrapped.emitStrategy(emitStrategy)); From 0895122d5b8f5968bb8025cff660fba1df6fe39c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 11:28:52 +0100 Subject: [PATCH 07/72] Add error handling --- .../main/java/com/bakdata/kafka/ConfiguredRepartitioned.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java index b8738bf2..85115e3c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java @@ -75,10 +75,11 @@ public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPa } Repartitioned configure(final Configurator configurator) { - return Repartitioned.as(this.name) + final Repartitioned repartitioned = Repartitioned.as(this.name) .withKeySerde(this.configureKeySerde(configurator)) .withValueSerde(this.configuredValueSerde(configurator)) - .withStreamPartitioner(this.streamPartitioner) + .withStreamPartitioner(this.streamPartitioner); + return this.numberOfPartitions == null ? repartitioned : repartitioned .withNumberOfPartitions(this.numberOfPartitions); } From 103a20ec124fa09e285acc0ca0ebad540bc5c6c2 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 11:48:37 +0100 Subject: [PATCH 08/72] Add error handling --- .../com/bakdata/kafka/ConfiguredStores.java | 86 +++++++++++++++++++ .../com/bakdata/kafka/TopologyBuilder.java | 4 + 2 files changed, 90 insertions(+) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java new file mode 100644 index 00000000..0c8093a5 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java @@ -0,0 +1,86 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.SessionBytesStoreSupplier; +import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.TimestampedKeyValueStore; +import org.apache.kafka.streams.state.TimestampedWindowStore; +import org.apache.kafka.streams.state.VersionedBytesStoreSupplier; +import org.apache.kafka.streams.state.VersionedKeyValueStore; +import org.apache.kafka.streams.state.WindowBytesStoreSupplier; +import org.apache.kafka.streams.state.WindowStore; + +@RequiredArgsConstructor +public class ConfiguredStores { + + private final @NonNull Configurator configurator; + + public StoreBuilder> sessionStoreBuilder(final SessionBytesStoreSupplier supplier, + final Preconfigured> keySerde, final Preconfigured> valueSerde) { + return Stores.sessionStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), + this.configurator.configureForValues(valueSerde)); + } + + public StoreBuilder> timestampedWindowStoreBuilder( + final WindowBytesStoreSupplier supplier, final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return Stores.timestampedWindowStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), + this.configurator.configureForValues(valueSerde)); + } + + public StoreBuilder> windowStoreBuilder(final WindowBytesStoreSupplier supplier, + final Preconfigured> keySerde, final Preconfigured> valueSerde) { + return Stores.windowStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), + this.configurator.configureForValues(valueSerde)); + } + + public StoreBuilder> versionedKeyValueStoreBuilder( + final VersionedBytesStoreSupplier supplier, final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return Stores.versionedKeyValueStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), + this.configurator.configureForValues(valueSerde)); + } + + public StoreBuilder> timestampedKeyValueStoreBuilder( + final KeyValueBytesStoreSupplier supplier, final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return Stores.timestampedKeyValueStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), + this.configurator.configureForValues(valueSerde)); + } + + public StoreBuilder> keyValueStoreBuilder(final KeyValueBytesStoreSupplier supplier, + final Preconfigured> keySerde, final Preconfigured> valueSerde) { + return Stores.keyValueStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), + this.configurator.configureForValues(valueSerde)); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index 34f90eb7..0a6ec451 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -229,6 +229,10 @@ public StreamsContext getContext() { return new StreamsContext(this.topics, this.createConfigurator()); } + public ConfiguredStores stores() { + return new ConfiguredStores(this.createConfigurator()); + } + Topology build() { return this.streamsBuilder.build(); } From 5780aefb7b65a5a88f026eaab64d48848c213139 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 27 Jan 2025 12:32:10 +0100 Subject: [PATCH 09/72] Add error handling --- .../com/bakdata/kafka/ConfiguredConsumed.java | 25 ++++++--------- .../bakdata/kafka/ConfiguredMaterialized.java | 28 +++++++---------- .../com/bakdata/kafka/ConfiguredProduced.java | 25 ++++++--------- .../kafka/ConfiguredRepartitioned.java | 31 +++++++++---------- 4 files changed, 46 insertions(+), 63 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java index ffda6052..d6308187 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java @@ -25,6 +25,7 @@ package com.bakdata.kafka; import lombok.AccessLevel; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.With; import org.apache.kafka.common.serialization.Serde; @@ -34,20 +35,20 @@ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class ConfiguredConsumed { +public final class ConfiguredConsumed { - private final Preconfigured> keySerde; - private final Preconfigured> valueSerde; + private final @NonNull Preconfigured> keySerde; + private final @NonNull Preconfigured> valueSerde; private final TimestampExtractor timestampExtractor; private final AutoOffsetReset offsetResetPolicy; private final String name; public static ConfiguredConsumed keySerde(final Preconfigured> keySerde) { - return with(keySerde, null); + return with(keySerde, Preconfigured.defaultSerde()); } public static ConfiguredConsumed valueSerde(final Preconfigured> valueSerde) { - return with(null, valueSerde); + return with(Preconfigured.defaultSerde(), valueSerde); } public static ConfiguredConsumed with(final Preconfigured> keySerde, @@ -56,22 +57,16 @@ public static ConfiguredConsumed with(final Preconfigured> } public static ConfiguredConsumed as(final String processorName) { - return new ConfiguredConsumed<>(null, null, null, null, processorName); + return new ConfiguredConsumed<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, + processorName); } Consumed configure(final Configurator configurator) { return Consumed.as(this.name) - .withKeySerde(this.configureKeySerde(configurator)) - .withValueSerde(this.configureValueSerde(configurator)) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) .withOffsetResetPolicy(this.offsetResetPolicy) .withTimestampExtractor(this.timestampExtractor); } - private Serde configureValueSerde(final Configurator configurator) { - return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); - } - - private Serde configureKeySerde(final Configurator configurator) { - return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); - } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java index b7d24640..622ed352 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.Map; import lombok.AccessLevel; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.With; import org.apache.kafka.common.serialization.Serde; @@ -36,12 +37,12 @@ import org.apache.kafka.streams.state.DslStoreSuppliers; @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class ConfiguredMaterialized { +public final class ConfiguredMaterialized { @With - private final Preconfigured> keySerde; + private final @NonNull Preconfigured> keySerde; @With - private final Preconfigured> valueSerde; + private final @NonNull Preconfigured> valueSerde; private final String storeName; @With private final Duration retention; @@ -53,12 +54,12 @@ public class ConfiguredMaterialized { public static ConfiguredMaterialized keySerde( final Preconfigured> keySerde) { - return with(keySerde, null); + return with(keySerde, Preconfigured.defaultSerde()); } public static ConfiguredMaterialized valueSerde( final Preconfigured> valueSerde) { - return with(null, valueSerde); + return with(Preconfigured.defaultSerde(), valueSerde); } public static ConfiguredMaterialized with( @@ -68,18 +69,20 @@ public static ConfiguredMaterialized with( } public static ConfiguredMaterialized as(final String storeName) { - return new ConfiguredMaterialized<>(null, null, storeName, null, null, new HashMap<>(), true, true); + return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeName, null, + null, new HashMap<>(), true, true); } public static ConfiguredMaterialized as( final DslStoreSuppliers storeSuppliers) { - return new ConfiguredMaterialized<>(null, null, null, null, storeSuppliers, new HashMap<>(), true, true); + return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, + storeSuppliers, new HashMap<>(), true, true); } Materialized configure(final Configurator configurator) { final Materialized materialized = Materialized.as(this.storeName) - .withKeySerde(this.configureKeySerde(configurator)) - .withValueSerde(this.configuredValueSerde(configurator)); + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)); if (this.retention != null) { materialized.withRetention(this.retention); } @@ -99,11 +102,4 @@ Materialized configure(final Configurator configurator) { return materialized; } - private Serde configuredValueSerde(final Configurator configurator) { - return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); - } - - private Serde configureKeySerde(final Configurator configurator) { - return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); - } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java index 62d0d161..32ccaaac 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java @@ -25,6 +25,7 @@ package com.bakdata.kafka; import lombok.AccessLevel; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.With; import org.apache.kafka.common.serialization.Serde; @@ -33,19 +34,19 @@ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class ConfiguredProduced { +public final class ConfiguredProduced { - private final Preconfigured> keySerde; - private final Preconfigured> valueSerde; + private final @NonNull Preconfigured> keySerde; + private final @NonNull Preconfigured> valueSerde; private final StreamPartitioner streamPartitioner; private final String name; public static ConfiguredProduced keySerde(final Preconfigured> keySerde) { - return with(keySerde, null); + return with(keySerde, Preconfigured.defaultSerde()); } public static ConfiguredProduced valueSerde(final Preconfigured> valueSerde) { - return with(null, valueSerde); + return with(Preconfigured.defaultSerde(), valueSerde); } public static ConfiguredProduced with(final Preconfigured> keySerde, @@ -54,21 +55,15 @@ public static ConfiguredProduced with(final Preconfigured> } public static ConfiguredProduced as(final String processorName) { - return new ConfiguredProduced<>(null, null, null, processorName); + return new ConfiguredProduced<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, + processorName); } Produced configure(final Configurator configurator) { return Produced.as(this.name) - .withKeySerde(this.configureKeySerde(configurator)) - .withValueSerde(this.configuredValueSerde(configurator)) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) .withStreamPartitioner(this.streamPartitioner); } - private Serde configuredValueSerde(final Configurator configurator) { - return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); - } - - private Serde configureKeySerde(final Configurator configurator) { - return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); - } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java index 85115e3c..3a713c41 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java @@ -25,6 +25,7 @@ package com.bakdata.kafka; import lombok.AccessLevel; +import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.With; import org.apache.kafka.common.serialization.Serde; @@ -32,12 +33,12 @@ import org.apache.kafka.streams.processor.StreamPartitioner; @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class ConfiguredRepartitioned { +public final class ConfiguredRepartitioned { @With - private final Preconfigured> keySerde; + private final @NonNull Preconfigured> keySerde; @With - private final Preconfigured> valueSerde; + private final @NonNull Preconfigured> valueSerde; @With private final StreamPartitioner streamPartitioner; @With @@ -45,11 +46,11 @@ public class ConfiguredRepartitioned { private final Integer numberOfPartitions; public static ConfiguredRepartitioned keySerde(final Preconfigured> keySerde) { - return with(keySerde, null); + return with(keySerde, Preconfigured.defaultSerde()); } public static ConfiguredRepartitioned valueSerde(final Preconfigured> valueSerde) { - return with(null, valueSerde); + return with(Preconfigured.defaultSerde(), valueSerde); } public static ConfiguredRepartitioned with(final Preconfigured> keySerde, @@ -58,15 +59,18 @@ public static ConfiguredRepartitioned with(final Preconfigured ConfiguredRepartitioned as(final String name) { - return new ConfiguredRepartitioned<>(null, null, null, name, null); + return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, name, + null); } public static ConfiguredRepartitioned numberOfPartitions(final int numberOfPartitions) { - return new ConfiguredRepartitioned<>(null, null, null, null, numberOfPartitions); + return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, + numberOfPartitions); } public static ConfiguredRepartitioned streamPartitioner(final StreamPartitioner partitioner) { - return new ConfiguredRepartitioned<>(null, null, partitioner, null, null); + return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), partitioner, + null, null); } public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPartitions) { @@ -76,18 +80,11 @@ public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPa Repartitioned configure(final Configurator configurator) { final Repartitioned repartitioned = Repartitioned.as(this.name) - .withKeySerde(this.configureKeySerde(configurator)) - .withValueSerde(this.configuredValueSerde(configurator)) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) .withStreamPartitioner(this.streamPartitioner); return this.numberOfPartitions == null ? repartitioned : repartitioned .withNumberOfPartitions(this.numberOfPartitions); } - private Serde configuredValueSerde(final Configurator configurator) { - return this.valueSerde == null ? null : configurator.configureForValues(this.valueSerde); - } - - private Serde configureKeySerde(final Configurator configurator) { - return this.keySerde == null ? null : configurator.configureForKeys(this.keySerde); - } } From 4fe684f4408fc36d2d9c0d44e559e51b2a4d5f0c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 13 Feb 2025 17:01:35 +0100 Subject: [PATCH 10/72] Add JavaDoc --- .../com/bakdata/kafka/ConfiguredConsumed.java | 35 +++++++++++ .../bakdata/kafka/ConfiguredMaterialized.java | 48 ++++++++++++++ .../com/bakdata/kafka/ConfiguredProduced.java | 35 +++++++++++ .../kafka/ConfiguredRepartitioned.java | 54 ++++++++++++++++ .../com/bakdata/kafka/ConfiguredStores.java | 63 +++++++++++++++++++ .../kafka/ImprovedBranchedKStream.java | 4 ++ .../kafka/ImprovedCogroupedKStream.java | 21 +++++++ .../bakdata/kafka/ImprovedKGroupedStream.java | 19 ++++++ 8 files changed, 279 insertions(+) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java index d6308187..e06f7c2f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java @@ -33,6 +33,12 @@ import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.processor.TimestampExtractor; +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Consumed} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Consumed + */ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class ConfiguredConsumed { @@ -43,19 +49,48 @@ public final class ConfiguredConsumed { private final AutoOffsetReset offsetResetPolicy; private final String name; + /** + * Create an instance of {@code ConfiguredConsumed} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredConsumed} + * @param type of keys + * @param type of values + */ public static ConfiguredConsumed keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } + /** + * Create an instance of {@code ConfiguredConsumed} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredConsumed} + * @param type of keys + * @param type of values + */ public static ConfiguredConsumed valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } + /** + * Create an instance of {@code ConfiguredConsumed} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredConsumed} + * @param type of keys + * @param type of values + */ public static ConfiguredConsumed with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { return new ConfiguredConsumed<>(keySerde, valueSerde, null, null, null); } + /** + * Create an instance of {@code ConfiguredConsumed} with provided processor name + * @param processorName the processor name to be used + * @return a new instance of {@code ConfiguredConsumed} + * @param type of keys + * @param type of values + */ public static ConfiguredConsumed as(final String processorName) { return new ConfiguredConsumed<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, processorName); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java index 622ed352..b246b102 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java @@ -36,6 +36,13 @@ import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.state.DslStoreSuppliers; +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Materialized} using {@link Configurator} + * @param type of keys + * @param type of values + * @param type of state store + * @see Materialized + */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class ConfiguredMaterialized { @@ -52,27 +59,68 @@ public final class ConfiguredMaterialized { private final boolean loggingEnabled; private final boolean cachingEnabled; + /** + * Create an instance of {@code ConfiguredMaterialized} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredMaterialized} + * @param type of keys + * @param type of values + * @param type of state store + */ public static ConfiguredMaterialized keySerde( final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } + /** + * Create an instance of {@code ConfiguredMaterialized} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredMaterialized} + * @param type of keys + * @param type of values + * @param type of state store + */ public static ConfiguredMaterialized valueSerde( final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } + /** + * Create an instance of {@code ConfiguredMaterialized} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredMaterialized} + * @param type of keys + * @param type of values + * @param type of state store + */ public static ConfiguredMaterialized with( final Preconfigured> keySerde, final Preconfigured> valueSerde) { return new ConfiguredMaterialized<>(keySerde, valueSerde, null, null, null, new HashMap<>(), true, true); } + /** + * Create an instance of {@code ConfiguredMaterialized} with provided store name + * @param storeName the store name to be used + * @return a new instance of {@code ConfiguredMaterialized} + * @param type of keys + * @param type of values + * @param type of state store + */ public static ConfiguredMaterialized as(final String storeName) { return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeName, null, null, new HashMap<>(), true, true); } + /** + * Create an instance of {@code ConfiguredMaterialized} with provided store suppliers + * @param storeSuppliers the store suppliers to be used + * @return a new instance of {@code ConfiguredMaterialized} + * @param type of keys + * @param type of values + * @param type of state store + */ public static ConfiguredMaterialized as( final DslStoreSuppliers storeSuppliers) { return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java index 32ccaaac..6d365b42 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java @@ -32,6 +32,12 @@ import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.processor.StreamPartitioner; +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Produced} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Produced + */ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class ConfiguredProduced { @@ -41,19 +47,48 @@ public final class ConfiguredProduced { private final StreamPartitioner streamPartitioner; private final String name; + /** + * Create an instance of {@code ConfiguredProduced} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredProduced} + * @param type of keys + * @param type of values + */ public static ConfiguredProduced keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } + /** + * Create an instance of {@code ConfiguredProduced} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredProduced} + * @param type of keys + * @param type of values + */ public static ConfiguredProduced valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } + /** + * Create an instance of {@code ConfiguredProduced} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredProduced} + * @param type of keys + * @param type of values + */ public static ConfiguredProduced with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { return new ConfiguredProduced<>(keySerde, valueSerde, null, null); } + /** + * Create an instance of {@code ConfiguredProduced} with provided processor name + * @param processorName the processor name to be used + * @return a new instance of {@code ConfiguredProduced} + * @param type of keys + * @param type of values + */ public static ConfiguredProduced as(final String processorName) { return new ConfiguredProduced<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, processorName); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java index 3a713c41..0c8a60d7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java @@ -32,6 +32,12 @@ import org.apache.kafka.streams.kstream.Repartitioned; import org.apache.kafka.streams.processor.StreamPartitioner; +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Repartitioned} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Repartitioned + */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) public final class ConfiguredRepartitioned { @@ -45,34 +51,82 @@ public final class ConfiguredRepartitioned { private final String name; private final Integer numberOfPartitions; + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredRepartitioned} + * @param type of keys + * @param type of values + */ public static ConfiguredRepartitioned keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredRepartitioned} + * @param type of keys + * @param type of values + */ public static ConfiguredRepartitioned valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredRepartitioned} + * @param type of keys + * @param type of values + */ public static ConfiguredRepartitioned with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { return new ConfiguredRepartitioned<>(keySerde, valueSerde, null, null, null); } + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided name + * @param name the name used as a processor name and part of the repartition topic name + * @return a new instance of {@code ConfiguredRepartitioned} + * @param type of keys + * @param type of values + */ public static ConfiguredRepartitioned as(final String name) { return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, name, null); } + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided number of partitions for repartition topic + * @param numberOfPartitions number of partitions + * @return a new instance of {@code ConfiguredRepartitioned} + * @param type of keys + * @param type of values + */ public static ConfiguredRepartitioned numberOfPartitions(final int numberOfPartitions) { return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, numberOfPartitions); } + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided partitioner for repartition topic + * @param partitioner partitioner to use + * @return a new instance of {@code ConfiguredRepartitioned} + * @param type of keys + * @param type of values + */ public static ConfiguredRepartitioned streamPartitioner(final StreamPartitioner partitioner) { return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), partitioner, null, null); } + /** + * Create an instance of {@code ConfiguredRepartitioned} with provided number of partitions for repartition topic + * @param numberOfPartitions number of partitions + * @return a new instance of {@code ConfiguredRepartitioned} + */ public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPartitions) { return new ConfiguredRepartitioned<>(this.keySerde, this.valueSerde, this.streamPartitioner, this.name, numberOfPartitions); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java index 0c8093a5..3593a58f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java @@ -40,17 +40,40 @@ import org.apache.kafka.streams.state.WindowBytesStoreSupplier; import org.apache.kafka.streams.state.WindowStore; +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Stores} using {@link Configurator} + */ @RequiredArgsConstructor public class ConfiguredStores { private final @NonNull Configurator configurator; + /** + * Creates a {@code StoreBuilder} that can be used to build a {@code SessionStore} + * @param supplier a {@code SessionBytesStoreSupplier} + * @param keySerde the key serde to use + * @param valueSerde the value serde to use + * @return a store builder + * @param key type + * @param value type + * @see Stores#sessionStoreBuilder(SessionBytesStoreSupplier, Serde, Serde) + */ public StoreBuilder> sessionStoreBuilder(final SessionBytesStoreSupplier supplier, final Preconfigured> keySerde, final Preconfigured> valueSerde) { return Stores.sessionStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), this.configurator.configureForValues(valueSerde)); } + /** + * Creates a {@code StoreBuilder} that can be used to build a {@code TimestampedWindowStore} + * @param supplier a {@code WindowBytesStoreSupplier} + * @param keySerde the key serde to use + * @param valueSerde the value serde to use + * @return a store builder + * @param key type + * @param value type + * @see Stores#timestampedWindowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) + */ public StoreBuilder> timestampedWindowStoreBuilder( final WindowBytesStoreSupplier supplier, final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -58,12 +81,32 @@ public StoreBuilder> timestampedWindowStoreB this.configurator.configureForValues(valueSerde)); } + /** + * Creates a {@code StoreBuilder} that can be used to build a {@code WindowStore} + * @param supplier a {@code WindowBytesStoreSupplier} + * @param keySerde the key serde to use + * @param valueSerde the value serde to use + * @return a store builder + * @param key type + * @param value type + * @see Stores#windowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) + */ public StoreBuilder> windowStoreBuilder(final WindowBytesStoreSupplier supplier, final Preconfigured> keySerde, final Preconfigured> valueSerde) { return Stores.windowStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), this.configurator.configureForValues(valueSerde)); } + /** + * Creates a {@code StoreBuilder} that can be used to build a {@code VersionedKeyValueStore} + * @param supplier a {@code VersionedBytesStoreSupplier} + * @param keySerde the key serde to use + * @param valueSerde the value serde to use + * @return a store builder + * @param key type + * @param value type + * @see Stores#versionedKeyValueStoreBuilder(VersionedBytesStoreSupplier, Serde, Serde) + */ public StoreBuilder> versionedKeyValueStoreBuilder( final VersionedBytesStoreSupplier supplier, final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -71,6 +114,16 @@ public StoreBuilder> versionedKeyValueStoreB this.configurator.configureForValues(valueSerde)); } + /** + * Creates a {@code StoreBuilder} that can be used to build a {@code TimestampedKeyValueStore} + * @param supplier a {@code KeyValueBytesStoreSupplier} + * @param keySerde the key serde to use + * @param valueSerde the value serde to use + * @return a store builder + * @param key type + * @param value type + * @see Stores#timestampedKeyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) + */ public StoreBuilder> timestampedKeyValueStoreBuilder( final KeyValueBytesStoreSupplier supplier, final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -78,6 +131,16 @@ public StoreBuilder> timestampedKeyValueSt this.configurator.configureForValues(valueSerde)); } + /** + * Creates a {@code StoreBuilder} that can be used to build a {@code KeyValueStore} + * @param supplier a {@code KeyValueBytesStoreSupplier} + * @param keySerde the key serde to use + * @param valueSerde the value serde to use + * @return a store builder + * @param key type + * @param value type + * @see Stores#keyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) + */ public StoreBuilder> keyValueStoreBuilder(final KeyValueBytesStoreSupplier supplier, final Preconfigured> keySerde, final Preconfigured> valueSerde) { return Stores.keyValueStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java index 9c19e4bf..5b083156 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java @@ -30,6 +30,10 @@ import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Predicate; +/** + * Extends the {@code BranchedKStream} interface by adding methods to simplify Serde configuration, error handling, + * and topic access + */ public interface ImprovedBranchedKStream extends BranchedKStream { @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java index 4d36833d..d439a61b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -37,6 +37,10 @@ import org.apache.kafka.streams.kstream.Windows; import org.apache.kafka.streams.state.KeyValueStore; +/** + * Extends the {@code CogroupedKStream} interface by adding methods to simplify Serde configuration, error handling, + * and topic access + */ public interface ImprovedCogroupedKStream extends CogroupedKStream { @Override @@ -53,6 +57,14 @@ ImprovedCogroupedKStream cogroup(KGroupedStream groupedSt ImprovedKTable aggregate(Initializer initializer, Materialized> materialized); + /** + * Aggregate the values of records in these streams by the grouped key + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @see #aggregate(Initializer, Materialized) + */ ImprovedKTable aggregate(Initializer initializer, ConfiguredMaterialized> materialized); @@ -60,6 +72,15 @@ ImprovedKTable aggregate(Initializer initializer, ImprovedKTable aggregate(Initializer initializer, Named named, Materialized> materialized); + /** + * Aggregate the values of records in these streams by the grouped key + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param named name the processors + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @see #aggregate(Initializer, Named, Materialized) + */ ImprovedKTable aggregate(Initializer initializer, Named named, ConfiguredMaterialized> materialized); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index 8c54bae1..e492d3cd 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -37,6 +37,10 @@ import org.apache.kafka.streams.kstream.Windows; import org.apache.kafka.streams.state.KeyValueStore; +/** + * Extends the {@code KGroupedStream} interface by adding methods to simplify Serde configuration, error handling, + * and topic access + */ public interface ImprovedKGroupedStream extends KGroupedStream { @Override @@ -48,11 +52,26 @@ public interface ImprovedKGroupedStream extends KGroupedStream { @Override ImprovedKTable count(Materialized> materialized); + /** + * Count the number of records in this stream by the grouped key + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys and {@code Long} values that + * represent the latest (rolling) count (i.e., number of records) for each key + * @see #count(Materialized) + */ ImprovedKTable count(ConfiguredMaterialized> materialized); @Override ImprovedKTable count(Named named, Materialized> materialized); + /** + * Count the number of records in this stream by the grouped key + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys and {@code Long} values that + * represent the latest (rolling) count (i.e., number of records) for each key + * @see #count(Named, Materialized) + */ ImprovedKTable count(Named named, ConfiguredMaterialized> materialized); From c0cdd70e8806a162179a4ba67e5b516cebdab81c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 09:30:19 +0100 Subject: [PATCH 11/72] Add docs --- .../bakdata/kafka/ImprovedKGroupedStream.java | 38 +++++++++ .../bakdata/kafka/ImprovedKGroupedTable.java | 67 ++++++++++++++++ .../com/bakdata/kafka/ImprovedKStream.java | 4 + .../com/bakdata/kafka/ImprovedKTable.java | 4 + ...provedSessionWindowedCogroupedKStream.java | 25 ++++++ .../kafka/ImprovedSessionWindowedKStream.java | 63 +++++++++++++++ .../ImprovedTimeWindowedCogroupedKStream.java | 23 ++++++ .../kafka/ImprovedTimeWindowedKStream.java | 61 +++++++++++++++ .../java/com/bakdata/kafka/KErrorStream.java | 25 ++++++ .../com/bakdata/kafka/StreamsContext.java | 78 +++++++++++++++++++ 10 files changed, 388 insertions(+) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index e492d3cd..df99ff25 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -81,6 +81,14 @@ ImprovedKTable count(Named named, @Override ImprovedKTable reduce(Reducer reducer, Materialized> materialized); + /** + * Combine the value of records in this stream by the grouped key + * @param reducer a {@code Reducer} that computes a new aggregate result + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @see #reduce(Reducer, Materialized) + */ ImprovedKTable reduce(Reducer reducer, ConfiguredMaterialized> materialized); @@ -88,6 +96,15 @@ ImprovedKTable reduce(Reducer reducer, ImprovedKTable reduce(Reducer reducer, Named named, Materialized> materialized); + /** + * Combine the value of records in this stream by the grouped key + * @param reducer a {@code Reducer} that computes a new aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @see #reduce(Reducer, Named, Materialized) + */ ImprovedKTable reduce(Reducer reducer, Named named, ConfiguredMaterialized> materialized); @@ -98,6 +115,16 @@ ImprovedKTable reduce(Reducer reducer, Named named, ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@code Aggregator} that computes a new aggregate result + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @param the value type of the resulting {@code KTable} + * @see #aggregate(Initializer, Aggregator, Materialized) + */ ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, ConfiguredMaterialized> materialized); @@ -105,6 +132,17 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, Named named, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@code Aggregator} that computes a new aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @param the value type of the resulting {@code KTable} + * @see #aggregate(Initializer, Aggregator, Named, Materialized) + */ ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, Named named, ConfiguredMaterialized> materialized); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java index 65aecfe2..6e8bf273 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -33,16 +33,37 @@ import org.apache.kafka.streams.kstream.Reducer; import org.apache.kafka.streams.state.KeyValueStore; +/** + * Extends the {@code KGroupedTable} interface by adding methods to simplify Serde configuration, error handling, and + * topic access + */ public interface ImprovedKGroupedTable extends KGroupedTable { @Override ImprovedKTable count(Materialized> materialized); + /** + * Count number of records of the original {@code KTable} that got mapped to the same key into a new instance of + * {@code KTable} + * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store + * @return a {@code KTable} that contains "update" records with unmodified keys and Long values that represent the + * latest (rolling) count (i.e., number of records) for each key + * @see #count(Materialized) + */ ImprovedKTable count(ConfiguredMaterialized> materialized); @Override ImprovedKTable count(Named named, Materialized> materialized); + /** + * Count number of records of the original {@code KTable} that got mapped to the same key into a new instance of + * {@code KTable} + * @param named the {@code Named} config used to name the processor in the topology + * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store + * @return a {@code KTable} that contains "update" records with unmodified keys and Long values that represent the + * latest (rolling) count (i.e., number of records) for each key + * @see #count(Named, Materialized) + */ ImprovedKTable count(Named named, ConfiguredMaterialized> materialized); @@ -56,6 +77,16 @@ ImprovedKTable count(Named named, ImprovedKTable reduce(Reducer adder, Reducer subtractor, Materialized> materialized); + /** + * Combine the value of records of the original {@code KTable} that got mapped to the same key into a new + * instance of {@code KTable} + * @param adder a {@code Reducer} that adds a new value to the aggregate result + * @param subtractor a {@code Reducer} that removed an old value from the aggregate result + * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @see #reduce(Reducer, Reducer, Materialized) + */ ImprovedKTable reduce(Reducer adder, Reducer subtractor, ConfiguredMaterialized> materialized); @@ -63,6 +94,17 @@ ImprovedKTable reduce(Reducer adder, Reducer subtractor, ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, Materialized> materialized); + /** + * Combine the value of records of the original {@code KTable} that got mapped to the same key into a new + * instance of {@code KTable} + * @param adder a {@code Reducer} that adds a new value to the aggregate result + * @param subtractor a {@code Reducer} that removed an old value from the aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @see #reduce(Reducer, Reducer, Named, Materialized) + */ ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, ConfiguredMaterialized> materialized); @@ -74,6 +116,18 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator subtractor, Materialized> materialized); + /** + * Aggregate the value of records of the original {@code KTable} that got mapped to the same key into a new + * instance of {@code KTable} + * @param initializer an {@code Initializer} that provides an initial aggregate result value + * @param adder an {@code Aggregator} that adds a new record to the aggregate result + * @param subtractor an {@code Aggregator} that removed an old record from the aggregate result + * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @param the value type of the aggregated {@code KTable} + * @see #aggregate(Initializer, Aggregator, Aggregator, Materialized) + */ ImprovedKTable aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, ConfiguredMaterialized> materialized); @@ -83,6 +137,19 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator subtractor, Named named, Materialized> materialized); + /** + * Aggregate the value of records of the original {@code KTable} that got mapped to the same key into a new + * instance of {@code KTable} + * @param initializer an {@code Initializer} that provides an initial aggregate result value + * @param adder an {@code Aggregator} that adds a new record to the aggregate result + * @param subtractor an {@code Aggregator} that removed an old record from the aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store + * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the + * latest (rolling) aggregate for each key + * @param the value type of the aggregated {@code KTable} + * @see #aggregate(Initializer, Aggregator, Aggregator, Named, Materialized) + */ ImprovedKTable aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named, ConfiguredMaterialized> materialized); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index c9bdb0c0..b086f1fa 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -51,6 +51,10 @@ import org.apache.kafka.streams.processor.api.ProcessorSupplier; import org.apache.kafka.streams.state.KeyValueStore; +/** + * Extends the {@code KStream} interface by adding methods to simplify Serde configuration, error handling, and topic + * access + */ public interface ImprovedKStream extends KStream { @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index 583da8b4..8e770077 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -41,6 +41,10 @@ import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.state.KeyValueStore; +/** + * Extends the {@code KTable} interface by adding methods to simplify Serde configuration, error handling, and topic + * access + */ public interface ImprovedKTable extends KTable { @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java index ea5002fa..9b1655cd 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -33,6 +33,10 @@ import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.state.SessionStore; +/** + * Extends the {@code SessionWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, + * error handling, and topic access + */ public interface ImprovedSessionWindowedCogroupedKStream extends SessionWindowedCogroupedKStream { @Override @@ -46,6 +50,16 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, Materialized> materialized); + /** + * Aggregate the values of records in these streams by the grouped key and defined sessions + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param sessionMerger a {@code Merger} that combines two aggregation results + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @see #aggregate(Initializer, Merger, Materialized) + */ ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, ConfiguredMaterialized> materialized); @@ -53,6 +67,17 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, Named named, Materialized> materialized); + /** + * Aggregate the values of records in these streams by the grouped key and defined sessions + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param sessionMerger a {@code Merger} that combines two aggregation results + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @see #aggregate(Initializer, Merger, Named, Materialized) + */ ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, Named named, ConfiguredMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java index 896ffffe..6352e2fe 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -36,6 +36,10 @@ import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.state.SessionStore; +/** + * Extends the {@code SessionWindowedKStream} interface by adding methods to simplify Serde configuration, error + * handling, and topic access + */ public interface ImprovedSessionWindowedKStream extends SessionWindowedKStream { @Override @@ -47,12 +51,29 @@ public interface ImprovedSessionWindowedKStream extends SessionWindowedKSt @Override ImprovedKTable, Long> count(Materialized> materialized); + /** + * Count the number of records in this stream by the grouped key and defined sessions + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values + * that represent + * the latest (rolling) count (i.e., number of records) for each key per session + * @see #count(Materialized) + */ ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); @Override ImprovedKTable, Long> count(Named named, Materialized> materialized); + /** + * Count the number of records in this stream by the grouped key and defined sessions + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values + * that represent + * the latest (rolling) count (i.e., number of records) for each key per session + * @see #count(Named, Materialized) + */ ImprovedKTable, Long> count(Named named, ConfiguredMaterialized> materialized); @@ -71,6 +92,17 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key and defined sessions + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@code Aggregator} that computes a new aggregate result + * @param sessionMerger a {@code Merger} that combines two aggregation results + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the latest (rolling) aggregate for each key per session + * @param the value type of the resulting {@code KTable} + * @see #aggregate(Initializer, Aggregator, Merger, Materialized) + */ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, @@ -82,6 +114,18 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Merger sessionMerger, Named named, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key and defined sessions + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@code Aggregator} that computes a new aggregate result + * @param sessionMerger a {@code Merger} that combines two aggregation results + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the latest (rolling) aggregate for each key per session + * @param the value type of the resulting {@code KTable} + * @see #aggregate(Initializer, Aggregator, Merger, Named, Materialized) + */ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named, @@ -97,6 +141,15 @@ ImprovedKTable, VR> aggregate(Initializer initializer, ImprovedKTable, V> reduce(Reducer reducer, Materialized> materialized); + /** + * Combine the values of records in this stream by the grouped key and defined sessions + * @param reducer a {@code Reducer} that computes a new aggregate result + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key per session + * @see #reduce(Reducer, Materialized) + */ ImprovedKTable, V> reduce(Reducer reducer, ConfiguredMaterialized> materialized); @@ -104,6 +157,16 @@ ImprovedKTable, V> reduce(Reducer reducer, ImprovedKTable, V> reduce(Reducer reducer, Named named, Materialized> materialized); + /** + * Combine the values of records in this stream by the grouped key and defined sessions + * @param reducer a {@code Reducer} that computes a new aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key per session + * @see #reduce(Reducer, Named, Materialized) + */ ImprovedKTable, V> reduce(Reducer reducer, Named named, ConfiguredMaterialized> materialized); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java index cbd90455..d578b097 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -32,6 +32,10 @@ import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.state.WindowStore; +/** + * Extends the {@code TimeWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, + * error handling, and topic access + */ public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindowedCogroupedKStream { @Override @@ -44,6 +48,15 @@ public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindo ImprovedKTable, VOut> aggregate(Initializer initializer, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key and defined windows + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @see #aggregate(Initializer, Materialized) + */ ImprovedKTable, VOut> aggregate(Initializer initializer, ConfiguredMaterialized> materialized); @@ -51,6 +64,16 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key and defined windows + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @see #aggregate(Initializer, Named, Materialized) + */ ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, ConfiguredMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java index e57ba7cc..87df2675 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -46,12 +46,31 @@ public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream, Long> count(Materialized> materialized); + /** + * Count the number of records in this stream by the grouped key and defined windows + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values + * that + * represent + * the latest (rolling) count (i.e., number of records) for each key within a window + * @see #count(Materialized) + */ ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); @Override ImprovedKTable, Long> count(Named named, Materialized> materialized); + /** + * Count the number of records in this stream by the grouped key and defined windows + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values + * that + * represent + * the latest (rolling) count (i.e., number of records) for each key within a window + * @see #count(Named, Materialized) + */ ImprovedKTable, Long> count(Named named, ConfiguredMaterialized> materialized); @@ -69,6 +88,17 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key and defined windows + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@code Aggregator} that computes a new aggregate result + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @param the value type of the resulting {@code KTable} + * @see #aggregate(Initializer, Aggregator, Materialized) + */ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, ConfiguredMaterialized> materialized); @@ -78,6 +108,18 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Named named, Materialized> materialized); + /** + * Aggregate the values of records in this stream by the grouped key and defined windows + * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result + * @param aggregator an {@code Aggregator} that computes a new aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @param the value type of the resulting {@code KTable} + * @see #aggregate(Initializer, Aggregator, Named, Materialized) + */ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Named named, ConfiguredMaterialized> materialized); @@ -92,6 +134,15 @@ ImprovedKTable, VR> aggregate(Initializer initializer, ImprovedKTable, V> reduce(Reducer reducer, Materialized> materialized); + /** + * Combine the values of records in this stream by the grouped key and defined windows + * @param reducer a {@code Reducer} that computes a new aggregate result + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @see #reduce(Reducer, Materialized) + */ ImprovedKTable, V> reduce(Reducer reducer, ConfiguredMaterialized> materialized); @@ -99,6 +150,16 @@ ImprovedKTable, V> reduce(Reducer reducer, ImprovedKTable, V> reduce(Reducer reducer, Named named, Materialized> materialized); + /** + * Combine the values of records in this stream by the grouped key and defined windows + * @param reducer a {@code Reducer} that computes a new aggregate result + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store + * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that + * represent the + * latest (rolling) aggregate for each key within a window + * @see #reduce(Reducer, Named, Materialized) + */ ImprovedKTable, V> reduce(Reducer reducer, Named named, ConfiguredMaterialized> materialized); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java index e2997dca..9af54689 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java @@ -26,13 +26,38 @@ import org.apache.kafka.streams.kstream.Named; +/** + * {@link ImprovedKStream} that contains successfully processed records and errors of a previous operation + * @param type of keys in the original {@link ImprovedKStream} + * @param type of values in the original {@link ImprovedKStream} + * @param type of keys in the processed {@link ImprovedKStream} + * @param type of values in the processed {@link ImprovedKStream} + */ public interface KErrorStream { + /** + * Get the stream of successfully processed values + * @return stream of processed values + */ ImprovedKStream values(); + /** + * Get the stream of successfully processed values + * @param named name of the processor + * @return stream of processed values + */ ImprovedKStream values(Named named); + /** + * Get the stream of errors that occurred during processing + * @return stream of errors + */ ImprovedKStream> errors(); + /** + * Get the stream of errors that occurred during processing + * @param named name of the processor + * @return stream of errors + */ ImprovedKStream> errors(Named named); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index 91a74930..ee4814cf 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -39,6 +39,10 @@ import org.apache.kafka.streams.kstream.TimeWindowedCogroupedKStream; import org.apache.kafka.streams.kstream.TimeWindowedKStream; +/** + * Provides context for the configured Kafka Streams application environment, i.e., topic configuration and + * StreamsConfig + */ @Value @Getter(AccessLevel.PACKAGE) public class StreamsContext { @@ -72,6 +76,13 @@ static KTable maybeUnwrap(final KTable table) { return table; } + /** + * Wrap a {@code KStream} and add methods to simplify Serde configuration, error handling, and topic access + * @param stream stream to be wrapped + * @return {@code ImprovedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedKStream wrap(final KStream stream) { if (stream instanceof ImprovedKStream) { return (ImprovedKStream) stream; @@ -79,6 +90,13 @@ public ImprovedKStream wrap(final KStream stream) { return new ImprovedKStreamImpl<>(stream, this); } + /** + * Wrap a {@code KGroupedStream} and add methods to simplify Serde configuration, error handling, and topic access + * @param stream stream to be wrapped + * @return {@code ImprovedKGroupedStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedKGroupedStream wrap(final KGroupedStream stream) { if (stream instanceof ImprovedKGroupedStream) { return (ImprovedKGroupedStream) stream; @@ -86,6 +104,14 @@ public ImprovedKGroupedStream wrap(final KGroupedStream return new ImprovedKGroupedStreamImpl<>(stream, this); } + /** + * Wrap a {@code TimeWindowedKStream} and add methods to simplify Serde configuration, error handling, and topic + * access + * @param stream stream to be wrapped + * @return {@code ImprovedTimeWindowedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedTimeWindowedKStream wrap( final TimeWindowedKStream stream) { if (stream instanceof ImprovedTimeWindowedKStream) { @@ -94,6 +120,14 @@ public ImprovedTimeWindowedKStream wrap( return new ImprovedTimeWindowedStreamImpl<>(stream, this); } + /** + * Wrap a {@code SessionWindowedKStream} and add methods to simplify Serde configuration, error handling, and + * topic access + * @param stream stream to be wrapped + * @return {@code ImprovedSessionWindowedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedSessionWindowedKStream wrap( final SessionWindowedKStream stream) { if (stream instanceof ImprovedSessionWindowedKStream) { @@ -102,6 +136,14 @@ public ImprovedSessionWindowedKStream wrap( return new ImprovedSessionWindowedStreamImpl<>(stream, this); } + /** + * Wrap a {@code TimeWindowedCogroupedKStream} and add methods to simplify Serde configuration, error handling, + * and topic access + * @param stream stream to be wrapped + * @return {@code ImprovedTimeWindowedCogroupedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedTimeWindowedCogroupedKStream wrap( final TimeWindowedCogroupedKStream stream) { if (stream instanceof ImprovedTimeWindowedCogroupedKStream) { @@ -110,6 +152,14 @@ public ImprovedTimeWindowedCogroupedKStream wrap( return new ImprovedTimeWindowedCogroupedStreamImpl<>(stream, this); } + /** + * Wrap a {@code SessionWindowedCogroupedKStream} and add methods to simplify Serde configuration, error + * handling, and topic access + * @param stream stream to be wrapped + * @return {@code ImprovedSessionWindowedCogroupedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedSessionWindowedCogroupedKStream wrap( final SessionWindowedCogroupedKStream stream) { if (stream instanceof ImprovedSessionWindowedCogroupedKStream) { @@ -118,6 +168,13 @@ public ImprovedSessionWindowedCogroupedKStream wrap( return new ImprovedSessionWindowedCogroupedStreamImpl<>(stream, this); } + /** + * Wrap a {@code CogroupedKStream} and add methods to simplify Serde configuration, error handling, and topic access + * @param stream stream to be wrapped + * @return {@code ImprovedCogroupedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedCogroupedKStream wrap(final CogroupedKStream stream) { if (stream instanceof ImprovedCogroupedKStream) { return (ImprovedCogroupedKStream) stream; @@ -125,6 +182,13 @@ public ImprovedCogroupedKStream wrap(final CogroupedKStream(stream, this); } + /** + * Wrap a {@code BranchedKStream} and add methods to simplify Serde configuration, error handling, and topic access + * @param stream stream to be wrapped + * @return {@code ImprovedBranchedKStream} + * @param type of keys in the stream + * @param type of values in the stream + */ public ImprovedBranchedKStream wrap(final BranchedKStream stream) { if (stream instanceof ImprovedBranchedKStream) { return (ImprovedBranchedKStream) stream; @@ -132,6 +196,13 @@ public ImprovedBranchedKStream wrap(final BranchedKStream(stream, this); } + /** + * Wrap a {@code KTable} and add methods to simplify Serde configuration, error handling, and topic access + * @param table table to be wrapped + * @return {@code ImprovedKTable} + * @param type of keys in the table + * @param type of values in the table + */ public ImprovedKTable wrap(final KTable table) { if (table instanceof ImprovedKTable) { return (ImprovedKTable) table; @@ -139,6 +210,13 @@ public ImprovedKTable wrap(final KTable table) { return new ImprovedKTableImpl<>(table, this); } + /** + * Wrap a {@code KGroupedTable} and add methods to simplify Serde configuration, error handling, and topic access + * @param table table to be wrapped + * @return {@code ImprovedKGroupedTable} + * @param type of keys in the table + * @param type of values in the table + */ public ImprovedKGroupedTable wrap(final KGroupedTable table) { if (table instanceof ImprovedKGroupedTable) { return (ImprovedKGroupedTable) table; From bb0ca14717612b184ad15a95c0caccd90a274cf6 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 09:58:24 +0100 Subject: [PATCH 12/72] Add docs --- .github/workflows/build-and-publish.yaml | 1 + .../kafka/ImprovedBranchedKStream.java | 2 + .../kafka/ImprovedCogroupedKStream.java | 2 + .../bakdata/kafka/ImprovedKGroupedStream.java | 2 + .../kafka/ImprovedKGroupedStreamImpl.java | 3 +- .../bakdata/kafka/ImprovedKGroupedTable.java | 2 + .../kafka/ImprovedKGroupedTableImpl.java | 12 ++++-- .../com/bakdata/kafka/ImprovedKStream.java | 2 + .../com/bakdata/kafka/ImprovedKTable.java | 2 + .../com/bakdata/kafka/ImprovedKTableImpl.java | 42 ++++++++++++------- ...provedSessionWindowedCogroupedKStream.java | 2 + ...vedSessionWindowedCogroupedStreamImpl.java | 9 ++-- .../kafka/ImprovedSessionWindowedKStream.java | 2 + .../ImprovedSessionWindowedStreamImpl.java | 6 ++- .../ImprovedTimeWindowedCogroupedKStream.java | 2 + .../kafka/ImprovedTimeWindowedKStream.java | 6 +++ .../com/bakdata/kafka/StreamsContext.java | 12 +++--- 17 files changed, 79 insertions(+), 30 deletions(-) diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml index 567baa4a..663472f3 100644 --- a/.github/workflows/build-and-publish.yaml +++ b/.github/workflows/build-and-publish.yaml @@ -4,6 +4,7 @@ on: push: tags: ["**"] branches: ["**"] + pull_request: jobs: build-and-publish: diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java index 5b083156..cacc1c3e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java @@ -33,6 +33,8 @@ /** * Extends the {@code BranchedKStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access + * @param type of keys + * @param type of values */ public interface ImprovedBranchedKStream extends BranchedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java index d439a61b..d18a5bec 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -40,6 +40,8 @@ /** * Extends the {@code CogroupedKStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access + * @param type of keys + * @param > type of values */ public interface ImprovedCogroupedKStream extends CogroupedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index df99ff25..5eac4328 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -40,6 +40,8 @@ /** * Extends the {@code KGroupedStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access + * @param type of keys + * @param > type of values */ public interface ImprovedKGroupedStream extends KGroupedStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java index 27014dc6..04399002 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -64,7 +64,8 @@ public ImprovedKTable count(final Materialized count(final ConfiguredMaterialized> materialized) { + public ImprovedKTable count( + final ConfiguredMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java index 6e8bf273..8206b6ef 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -36,6 +36,8 @@ /** * Extends the {@code KGroupedTable} interface by adding methods to simplify Serde configuration, error handling, and * topic access + * @param type of keys + * @param > type of values */ public interface ImprovedKGroupedTable extends KGroupedTable { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java index 76089fe6..4fa7b97d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java @@ -47,7 +47,8 @@ public ImprovedKTable count(final Materialized count(final ConfiguredMaterialized> materialized) { + public ImprovedKTable count( + final ConfiguredMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -111,7 +112,8 @@ public ImprovedKTable aggregate(final Initializer initializer, } @Override - public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator adder, final Aggregator subtractor, final ConfiguredMaterialized> materialized) { return this.aggregate(initializer, adder, subtractor, materialized.configure(this.context.getConfigurator())); @@ -126,10 +128,12 @@ public ImprovedKTable aggregate(final Initializer initializer, } @Override - public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, + public ImprovedKTable aggregate(final Initializer initializer, + final Aggregator adder, final Aggregator subtractor, final Named named, final ConfiguredMaterialized> materialized) { - return this.aggregate(initializer, adder, subtractor, named, materialized.configure(this.context.getConfigurator())); + return this.aggregate(initializer, adder, subtractor, named, + materialized.configure(this.context.getConfigurator())); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index b086f1fa..84e3dfa5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -54,6 +54,8 @@ /** * Extends the {@code KStream} interface by adding methods to simplify Serde configuration, error handling, and topic * access + * @param type of keys + * @param > type of values */ public interface ImprovedKStream extends KStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index 8e770077..fc18130a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -44,6 +44,8 @@ /** * Extends the {@code KTable} interface by adding methods to simplify Serde configuration, error handling, and topic * access + * @param type of keys + * @param > type of values */ public interface ImprovedKTable extends KTable { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java index f7f21249..efe30688 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -240,8 +240,10 @@ public ImprovedKTable transformValues( @Override public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, - final ConfiguredMaterialized> materialized, final String... stateStoreNames) { - return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), stateStoreNames); + final ConfiguredMaterialized> materialized, + final String... stateStoreNames) { + return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), + stateStoreNames); } @Override @@ -439,8 +441,10 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, - final ValueJoiner joiner, final ConfiguredMaterialized> materialized) { + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, + final ConfiguredMaterialized> materialized) { return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } @@ -454,10 +458,12 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final ConfiguredMaterialized> materialized) { - return this.join(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); + return this.join(other, foreignKeyExtractor, joiner, named, + materialized.configure(this.context.getConfigurator())); } @Override @@ -470,10 +476,12 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, + public ImprovedKTable join(final KTable other, + final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final ConfiguredMaterialized> materialized) { - return this.join(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); + return this.join(other, foreignKeyExtractor, joiner, tableJoined, + materialized.configure(this.context.getConfigurator())); } @Override @@ -509,9 +517,12 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, - final ValueJoiner joiner, final ConfiguredMaterialized> materialized) { - return this.leftJoin(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, + final ValueJoiner joiner, + final ConfiguredMaterialized> materialized) { + return this.leftJoin(other, foreignKeyExtractor, joiner, + materialized.configure(this.context.getConfigurator())); } @Override @@ -524,10 +535,12 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final ConfiguredMaterialized> materialized) { - return this.leftJoin(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); + return this.leftJoin(other, foreignKeyExtractor, joiner, named, + materialized.configure(this.context.getConfigurator())); } @Override @@ -541,7 +554,8 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, + public ImprovedKTable leftJoin(final KTable other, + final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final ConfiguredMaterialized> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java index 9b1655cd..2eaaa61f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -36,6 +36,8 @@ /** * Extends the {@code SessionWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access + * @param type of keys + * @param > type of values */ public interface ImprovedSessionWindowedCogroupedKStream extends SessionWindowedCogroupedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java index ec73b63b..f589076e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java @@ -62,7 +62,8 @@ public ImprovedKTable, V> aggregate(final Initializer initializer } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Merger sessionMerger, final ConfiguredMaterialized> materialized) { return this.aggregate(initializer, sessionMerger, materialized.configure(this.context.getConfigurator())); } @@ -75,8 +76,10 @@ public ImprovedKTable, V> aggregate(final Initializer initializer } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, + public ImprovedKTable, V> aggregate(final Initializer initializer, + final Merger sessionMerger, final Named named, final ConfiguredMaterialized> materialized) { - return this.aggregate(initializer, sessionMerger, named, materialized.configure(this.context.getConfigurator())); + return this.aggregate(initializer, sessionMerger, named, + materialized.configure(this.context.getConfigurator())); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java index 6352e2fe..b72498e5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -39,6 +39,8 @@ /** * Extends the {@code SessionWindowedKStream} interface by adding methods to simplify Serde configuration, error * handling, and topic access + * @param type of keys + * @param > type of values */ public interface ImprovedSessionWindowedKStream extends SessionWindowedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java index b330d075..268d722d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java @@ -102,7 +102,8 @@ public ImprovedKTable, VR> aggregate(final Initializer init public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final ConfiguredMaterialized> materialized) { - return this.aggregate(initializer, aggregator, sessionMerger, materialized.configure(this.context.getConfigurator())); + return this.aggregate(initializer, aggregator, sessionMerger, + materialized.configure(this.context.getConfigurator())); } @Override @@ -115,7 +116,8 @@ public ImprovedKTable, VR> aggregate(final Initializer init @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, - final Aggregator aggregator, final Merger sessionMerger, final Named named, + final Aggregator aggregator, final Merger sessionMerger, + final Named named, final ConfiguredMaterialized> materialized) { return this.aggregate(initializer, aggregator, sessionMerger, named, materialized.configure(this.context.getConfigurator())); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java index d578b097..1a9d3b3a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -35,6 +35,8 @@ /** * Extends the {@code TimeWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access + * @param type of keys + * @param > type of values */ public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindowedCogroupedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java index 87df2675..6050401b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -35,6 +35,12 @@ import org.apache.kafka.streams.kstream.Windowed; import org.apache.kafka.streams.state.WindowStore; +/** + * Extends the {@code TimeWindowedKStream} interface by adding methods to simplify Serde configuration, + * error handling, and topic access + * @param type of keys + * @param > type of values + */ public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream { @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index ee4814cf..97acd1de 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -54,24 +54,24 @@ public class StreamsContext { static KStream maybeUnwrap(final KStream stream) { if (stream instanceof ImprovedKStreamImpl) { - return ((ImprovedKStreamImpl) stream).getWrapped(); // Kafka Streams internally casts KStream to - // KStreamImpl in some cases + // Kafka Streams internally casts KStream to KStreamImpl in some cases + return ((ImprovedKStreamImpl) stream).getWrapped(); } return stream; } static KGroupedStream maybeUnwrap(final KGroupedStream stream) { if (stream instanceof ImprovedKGroupedStream) { - return ((ImprovedKGroupedStreamImpl) stream).getWrapped(); // Kafka Streams internally casts - // KGroupedStream to KGroupedStreamImpl in some cases + // Kafka Streams internally casts KGroupedStream to KGroupedStreamImpl in some cases + return ((ImprovedKGroupedStreamImpl) stream).getWrapped(); } return stream; } static KTable maybeUnwrap(final KTable table) { if (table instanceof ImprovedKTableImpl) { - return ((ImprovedKTableImpl) table).getWrapped(); // Kafka Streams internally casts KTable to - // KTableImpl in some cases + // Kafka Streams internally casts KTable to KTableImpl in some cases + return ((ImprovedKTableImpl) table).getWrapped(); } return table; } From b89c2ac45930aa29488606750bc96a340a4b3278 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 10:09:47 +0100 Subject: [PATCH 13/72] Add docs --- .../main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java | 2 +- .../src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java | 2 +- .../src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java | 2 +- .../src/main/java/com/bakdata/kafka/ImprovedKStream.java | 2 +- .../src/main/java/com/bakdata/kafka/ImprovedKTable.java | 2 +- .../bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java | 2 +- .../java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java | 2 +- .../com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java | 2 +- .../java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java index d18a5bec..5a183ac7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -41,7 +41,7 @@ * Extends the {@code CogroupedKStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedCogroupedKStream extends CogroupedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index 5eac4328..70d1c72f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -41,7 +41,7 @@ * Extends the {@code KGroupedStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedKGroupedStream extends KGroupedStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java index 8206b6ef..dd92292b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -37,7 +37,7 @@ * Extends the {@code KGroupedTable} interface by adding methods to simplify Serde configuration, error handling, and * topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedKGroupedTable extends KGroupedTable { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 84e3dfa5..099f8fa2 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -55,7 +55,7 @@ * Extends the {@code KStream} interface by adding methods to simplify Serde configuration, error handling, and topic * access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedKStream extends KStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index fc18130a..8440e7d4 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -45,7 +45,7 @@ * Extends the {@code KTable} interface by adding methods to simplify Serde configuration, error handling, and topic * access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedKTable extends KTable { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java index 2eaaa61f..0f52e85c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -37,7 +37,7 @@ * Extends the {@code SessionWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedSessionWindowedCogroupedKStream extends SessionWindowedCogroupedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java index b72498e5..ca087433 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -40,7 +40,7 @@ * Extends the {@code SessionWindowedKStream} interface by adding methods to simplify Serde configuration, error * handling, and topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedSessionWindowedKStream extends SessionWindowedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java index 1a9d3b3a..19f4affe 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -36,7 +36,7 @@ * Extends the {@code TimeWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindowedCogroupedKStream { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java index 6050401b..69d3ffd6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -39,7 +39,7 @@ * Extends the {@code TimeWindowedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access * @param type of keys - * @param > type of values + * @param type of values */ public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream { From eb012c9f6af0a6d4b64375bcbcf90c294fd7d200 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 10:40:48 +0100 Subject: [PATCH 14/72] Add tests --- .../com/bakdata/kafka/TopologyBuilder.java | 111 +++++++++ .../bakdata/kafka/ImprovedKStreamTest.java | 221 ++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index 0a6ec451..b89d5b05 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -49,36 +49,128 @@ public class TopologyBuilder { @NonNull Map kafkaProperties; + /** + * Create a {@code KStream} from the specified topic + * @param topic the topic name + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(String) + */ public ImprovedKStream stream(final String topic) { return this.getContext().wrap(this.streamsBuilder.stream(topic)); } + /** + * Create a {@code KStream} from the specified topic + * @param topic the topic name + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(String, Consumed) + */ public ImprovedKStream stream(final String topic, final Consumed consumed) { return this.getContext().wrap(this.streamsBuilder.stream(topic, consumed)); } + /** + * Create a {@code KStream} from the specified topic + * @param topic the topic name + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(String, Consumed) + */ + public ImprovedKStream stream(final String topic, final ConfiguredConsumed consumed) { + return this.stream(topic, consumed.configure(this.createConfigurator())); + } + + /** + * Create a {@code KStream} from the specified topics + * @param topics the topic names + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(Collection) + */ public ImprovedKStream stream(final Collection topics) { return this.getContext().wrap(this.streamsBuilder.stream(topics)); } + /** + * Create a {@code KStream} from the specified topics + * @param topics the topic names + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(Collection, Consumed) + */ public ImprovedKStream stream(final Collection topics, final Consumed consumed) { return this.getContext().wrap(this.streamsBuilder.stream(topics, consumed)); } + /** + * Create a {@code KStream} from the specified topics + * @param topics the topic names + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(Collection, Consumed) + */ + public ImprovedKStream stream(final Collection topics, + final ConfiguredConsumed consumed) { + return this.stream(topics, consumed.configure(this.createConfigurator())); + } + + /** + * Create a {@code KStream} from the specified topic pattern + * @param topicPattern the pattern to match for topic names + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(Pattern) + */ public ImprovedKStream stream(final Pattern topicPattern) { return this.getContext().wrap(this.streamsBuilder.stream(topicPattern)); } + /** + * Create a {@code KStream} from the specified topic pattern + * @param topicPattern the pattern to match for topic names + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(Pattern, Consumed) + */ public ImprovedKStream stream(final Pattern topicPattern, final Consumed consumed) { return this.getContext().wrap(this.streamsBuilder.stream(topicPattern, consumed)); } + /** + * Create a {@code KStream} from the specified topic pattern + * @param topicPattern the pattern to match for topic names + * @param consumed define optional parameters for streaming topics + * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @param type of keys + * @param type of values + * @see StreamsBuilder#stream(Pattern, Consumed) + */ + public ImprovedKStream stream(final Pattern topicPattern, final ConfiguredConsumed consumed) { + return this.stream(topicPattern, consumed.configure(this.createConfigurator())); + } + /** * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics()} * @param consumed define optional parameters for streaming topics * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream streamInput(final Consumed consumed) { return this.stream(this.topics.getInputTopics(), consumed); @@ -90,6 +182,7 @@ public ImprovedKStream streamInput(final Consumed consumed) { * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream streamInput(final ConfiguredConsumed consumed) { return this.streamInput(consumed.configure(this.createConfigurator())); @@ -100,6 +193,7 @@ public ImprovedKStream streamInput(final ConfiguredConsumed c * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Collection) */ public ImprovedKStream streamInput() { return this.stream(this.topics.getInputTopics()); @@ -112,6 +206,7 @@ public ImprovedKStream streamInput() { * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics(String)} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream streamInput(final String label, final Consumed consumed) { return this.stream(this.topics.getInputTopics(label), consumed); @@ -124,6 +219,7 @@ public ImprovedKStream streamInput(final String label, final Consum * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics(String)} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream streamInput(final String label, final ConfiguredConsumed consumed) { return this.streamInput(label, consumed.configure(this.createConfigurator())); @@ -135,6 +231,7 @@ public ImprovedKStream streamInput(final String label, final Config * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics(String)} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Collection) */ public ImprovedKStream streamInput(final String label) { return this.stream(this.topics.getInputTopics(label)); @@ -146,6 +243,7 @@ public ImprovedKStream streamInput(final String label) { * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream streamInputPattern(final Consumed consumed) { return this.stream(this.topics.getInputPattern(), consumed); @@ -157,6 +255,7 @@ public ImprovedKStream streamInputPattern(final Consumed cons * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream streamInputPattern(final ConfiguredConsumed consumed) { return this.streamInputPattern(consumed.configure(this.createConfigurator())); @@ -167,6 +266,7 @@ public ImprovedKStream streamInputPattern(final ConfiguredConsumed< * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Pattern) */ public ImprovedKStream streamInputPattern() { return this.stream(this.topics.getInputPattern()); @@ -179,6 +279,7 @@ public ImprovedKStream streamInputPattern() { * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream streamInputPattern(final String label, final Consumed consumed) { return this.stream(this.topics.getInputPattern(label), consumed); @@ -191,6 +292,7 @@ public ImprovedKStream streamInputPattern(final String label, final * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream streamInputPattern(final String label, final ConfiguredConsumed consumed) { @@ -203,6 +305,7 @@ public ImprovedKStream streamInputPattern(final String label, * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values + * @see StreamsBuilder#stream(Pattern) */ public ImprovedKStream streamInputPattern(final String label) { return this.stream(this.topics.getInputPattern(label)); @@ -225,10 +328,18 @@ public EffectiveAppConfiguration createEffectiveConfiguratio return new EffectiveAppConfiguration<>(this.topics, this.kafkaProperties); } + /** + * Create a {@code StreamsContext} to wrap Kafka Streams interfaces + * @return {@code StreamsContext} + */ public StreamsContext getContext() { return new StreamsContext(this.topics, this.createConfigurator()); } + /** + * Create stores using application context to lazily configures Serdes + * @return {@code ConfiguredStores} + */ public ConfiguredStores stores() { return new ConfiguredStores(this.createConfigurator()); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java new file mode 100644 index 00000000..732e3634 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -0,0 +1,221 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.util.Map; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.Serdes.StringSerde; +import org.junit.jupiter.api.Test; + +class ImprovedKStreamTest { + + @Test + void shouldWriteToOutput() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.toOutputTopic(); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .outputTopic("output") + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldWriteToOutputUsingProduced() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.toOutputTopic(ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .outputTopic("output") + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldWriteToLabeledOutput() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.toOutputTopic("label"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .labeledOutputTopics(Map.of("label", "output")) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldWriteToLabeledOutputUsingProduced() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.toOutputTopic("label", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .labeledOutputTopics(Map.of("label", "output")) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldWriteToError() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.toErrorTopic(); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .errorTopic("error") + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldWriteToErrorUsingProduced() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.toErrorTopic(ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .errorTopic("error") + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + private abstract static class SimpleApp implements StreamsApp { + + @Override + public String getUniqueAppId(final StreamsTopicConfig topics) { + return "my-app"; + } + + @Override + public SerdeConfig defaultSerializationConfig() { + return new SerdeConfig(StringSerde.class, StringSerde.class); + } + } +} From 2452aece4c5a4e2d8d16b92dba2d01bb820323ec Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 10:53:12 +0100 Subject: [PATCH 15/72] Add tests --- .../com/bakdata/kafka/ImprovedKStream.java | 5 + .../bakdata/kafka/ImprovedKStreamImpl.java | 10 + .../bakdata/kafka/ImprovedKStreamTest.java | 14 - .../java/com/bakdata/kafka/SimpleApp.java | 40 +++ .../bakdata/kafka/TopologyBuilderTest.java | 268 ++++++++++++++++++ 5 files changed, 323 insertions(+), 14 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 099f8fa2..82a27fe5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -47,6 +47,7 @@ import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.kstream.ValueTransformerSupplier; import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.processor.TopicNameExtractor; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.ProcessorSupplier; import org.apache.kafka.streams.state.KeyValueStore; @@ -242,6 +243,10 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKStream repartition(ConfiguredRepartitioned repartitioned); + void to(String topic, ConfiguredProduced produced); + + void to(TopicNameExtractor topicExtractor, ConfiguredProduced produced); + void toOutputTopic(); void toOutputTopic(Produced produced); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 50bc2750..d70eda3f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -433,6 +433,11 @@ public void to(final String topic, final Produced produced) { this.wrapped.to(topic, produced); } + @Override + public void to(final String topic, final ConfiguredProduced produced) { + this.to(topic, produced.configure(this.context.getConfigurator())); + } + @Override public void to(final TopicNameExtractor topicExtractor) { this.wrapped.to(topicExtractor); @@ -443,6 +448,11 @@ public void to(final TopicNameExtractor topicExtractor, final Produced topicExtractor, final ConfiguredProduced produced) { + this.to(topicExtractor, produced.configure(this.context.getConfigurator())); + } + @Override public void toOutputTopic() { this.to(this.context.getTopics().getOutputTopic()); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index 732e3634..f746bccb 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -27,7 +27,6 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.util.Map; import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.common.serialization.Serdes.StringSerde; import org.junit.jupiter.api.Test; class ImprovedKStreamTest { @@ -205,17 +204,4 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.stop(); } - - private abstract static class SimpleApp implements StreamsApp { - - @Override - public String getUniqueAppId(final StreamsTopicConfig topics) { - return "my-app"; - } - - @Override - public SerdeConfig defaultSerializationConfig() { - return new SerdeConfig(StringSerde.class, StringSerde.class); - } - } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java new file mode 100644 index 00000000..2a01ba64 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java @@ -0,0 +1,40 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.common.serialization.Serdes.StringSerde; + +abstract class SimpleApp implements StreamsApp { + + @Override + public String getUniqueAppId(final StreamsTopicConfig topics) { + return "my-app"; + } + + @Override + public SerdeConfig defaultSerializationConfig() { + return new SerdeConfig(StringSerde.class, StringSerde.class); + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java new file mode 100644 index 00000000..fec40988 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -0,0 +1,268 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import org.apache.kafka.common.serialization.Serdes; +import org.junit.jupiter.api.Test; + +class TopologyBuilderTest { + + @Test + void shouldReadFromInput() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInput(); + input.to("output"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromInputUsingConsumed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInput( + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromLabeledInput() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInput("label"); + input.to("output"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .labeledInputTopics(Map.of("label", List.of("input"))) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromLabeledInputUsingConsumed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInput("label", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .labeledInputTopics(Map.of("label", List.of("input"))) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromPatternInput() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInputPattern(); + input.to("output"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputPattern(Pattern.compile("input\\d+")) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input("input1").add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromPatternInputUsingConsumed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInputPattern( + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .inputPattern(Pattern.compile("input\\d+")) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input("input1") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromLabeledPatternInput() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInputPattern("label"); + input.to("output"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input("input1").add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReadFromLabeledPatternInputUsingConsumed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.streamInputPattern("label", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() + .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) + .build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input("input1") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + +} From dcdbe909f2b1d053001418129cf11f2fc48400fb Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 11:09:02 +0100 Subject: [PATCH 16/72] Add tests --- .../bakdata/kafka/ImprovedKStreamImpl.java | 107 +++---- .../bakdata/kafka/ImprovedKStreamTest.java | 261 ++++++++++++++++++ 2 files changed, 320 insertions(+), 48 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index d70eda3f..47f767b9 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -105,14 +105,14 @@ public ImprovedKStream map( @Override public KErrorStream mapCapturingErrors( final KeyValueMapper> mapper) { - return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper)); + return this.mapCapturingErrorsInternal(ErrorCapturingKeyValueMapper.captureErrors(mapper)); } @Override public KErrorStream mapCapturingErrors( final KeyValueMapper> mapper, final java.util.function.Predicate errorFilter) { - return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper, errorFilter)); + return this.mapCapturingErrorsInternal(ErrorCapturingKeyValueMapper.captureErrors(mapper, errorFilter)); } @Override @@ -126,14 +126,14 @@ public ImprovedKStream map( public KErrorStream mapCapturingErrors( final KeyValueMapper> mapper, final Named named) { - return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper), named); + return this.mapCapturingErrorsInternal(ErrorCapturingKeyValueMapper.captureErrors(mapper), named); } @Override public KErrorStream mapCapturingErrors( final KeyValueMapper> mapper, final java.util.function.Predicate errorFilter, final Named named) { - return this.mapCapturingErrors_(ErrorCapturingKeyValueMapper.captureErrors(mapper, errorFilter), named); + return this.mapCapturingErrorsInternal(ErrorCapturingKeyValueMapper.captureErrors(mapper, errorFilter), named); } @Override @@ -144,13 +144,13 @@ public ImprovedKStream mapValues(final ValueMapper KErrorStream mapValuesCapturingErrors( final ValueMapper mapper) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper)); + return this.mapValuesCapturingErrorsInternal(ErrorCapturingValueMapper.captureErrors(mapper)); } @Override public KErrorStream mapValuesCapturingErrors(final ValueMapper mapper, final java.util.function.Predicate errorFilter) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper, errorFilter)); + return this.mapValuesCapturingErrorsInternal(ErrorCapturingValueMapper.captureErrors(mapper, errorFilter)); } @Override @@ -161,13 +161,14 @@ public ImprovedKStream mapValues(final ValueMapper KErrorStream mapValuesCapturingErrors(final ValueMapper mapper, final Named named) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper), named); + return this.mapValuesCapturingErrorsInternal(ErrorCapturingValueMapper.captureErrors(mapper), named); } @Override public KErrorStream mapValuesCapturingErrors(final ValueMapper mapper, final java.util.function.Predicate errorFilter, final Named named) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapper.captureErrors(mapper, errorFilter), named); + return this.mapValuesCapturingErrorsInternal(ErrorCapturingValueMapper.captureErrors(mapper, errorFilter), + named); } @Override @@ -178,14 +179,15 @@ public ImprovedKStream mapValues(final ValueMapperWithKey KErrorStream mapValuesCapturingErrors( final ValueMapperWithKey mapper) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper)); + return this.mapValuesCapturingErrorsInternal(ErrorCapturingValueMapperWithKey.captureErrors(mapper)); } @Override public KErrorStream mapValuesCapturingErrors( final ValueMapperWithKey mapper, final java.util.function.Predicate errorFilter) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper, errorFilter)); + return this.mapValuesCapturingErrorsInternal( + ErrorCapturingValueMapperWithKey.captureErrors(mapper, errorFilter)); } @Override @@ -197,14 +199,15 @@ public ImprovedKStream mapValues(final ValueMapperWithKey KErrorStream mapValuesCapturingErrors( final ValueMapperWithKey mapper, final Named named) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper), named); + return this.mapValuesCapturingErrorsInternal(ErrorCapturingValueMapperWithKey.captureErrors(mapper), named); } @Override public KErrorStream mapValuesCapturingErrors( final ValueMapperWithKey mapper, final java.util.function.Predicate errorFilter, final Named named) { - return this.mapValuesCapturingErrors_(ErrorCapturingValueMapperWithKey.captureErrors(mapper, errorFilter), + return this.mapValuesCapturingErrorsInternal( + ErrorCapturingValueMapperWithKey.captureErrors(mapper, errorFilter), named); } @@ -219,7 +222,7 @@ public ImprovedKStream flatMap( public KErrorStream flatMapCapturingErrors( final KeyValueMapper>> mapper) { - return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper)); + return this.flatMapCapturingErrorsInternal(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper)); } @Override @@ -227,7 +230,7 @@ public KErrorStream flatMapCapturingErrors( final KeyValueMapper>> mapper, final java.util.function.Predicate errorFilter) { - return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper, errorFilter)); + return this.flatMapCapturingErrorsInternal(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper, errorFilter)); } @Override @@ -243,7 +246,7 @@ public KErrorStream flatMapCapturingErrors( final KeyValueMapper>> mapper, final Named named) { - return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper), named); + return this.flatMapCapturingErrorsInternal(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper), named); } @Override @@ -251,7 +254,8 @@ public KErrorStream flatMapCapturingErrors( final KeyValueMapper>> mapper, final java.util.function.Predicate errorFilter, final Named named) { - return this.flatMapCapturingErrors_(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper, errorFilter), named); + return this.flatMapCapturingErrorsInternal(ErrorCapturingFlatKeyValueMapper.captureErrors(mapper, errorFilter), + named); } @Override @@ -263,14 +267,15 @@ public ImprovedKStream flatMapValues( @Override public KErrorStream flatMapValuesCapturingErrors( final ValueMapper> mapper) { - return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper)); + return this.flatMapValuesCapturingErrorsInternal(ErrorCapturingFlatValueMapper.captureErrors(mapper)); } @Override public KErrorStream flatMapValuesCapturingErrors( final ValueMapper> mapper, final java.util.function.Predicate errorFilter) { - return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper, errorFilter)); + return this.flatMapValuesCapturingErrorsInternal( + ErrorCapturingFlatValueMapper.captureErrors(mapper, errorFilter)); } @Override @@ -283,7 +288,7 @@ public ImprovedKStream flatMapValues( @Override public KErrorStream flatMapValuesCapturingErrors( final ValueMapper> mapper, final Named named) { - return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper), named); + return this.flatMapValuesCapturingErrorsInternal(ErrorCapturingFlatValueMapper.captureErrors(mapper), named); } @Override @@ -291,7 +296,8 @@ public KErrorStream flatMapValuesCapturingErrors( final ValueMapper> mapper, final java.util.function.Predicate errorFilter, final Named named) { - return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapper.captureErrors(mapper, errorFilter), + return this.flatMapValuesCapturingErrorsInternal( + ErrorCapturingFlatValueMapper.captureErrors(mapper, errorFilter), named); } @@ -304,14 +310,14 @@ public ImprovedKStream flatMapValues( @Override public KErrorStream flatMapValuesCapturingErrors( final ValueMapperWithKey> mapper) { - return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper)); + return this.flatMapValuesCapturingErrorsInternal(ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper)); } @Override public KErrorStream flatMapValuesCapturingErrors( final ValueMapperWithKey> mapper, final java.util.function.Predicate errorFilter) { - return this.flatMapValuesCapturingErrors_( + return this.flatMapValuesCapturingErrorsInternal( ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper, errorFilter)); } @@ -326,14 +332,15 @@ public ImprovedKStream flatMapValues( public KErrorStream flatMapValuesCapturingErrors( final ValueMapperWithKey> mapper, final Named named) { - return this.flatMapValuesCapturingErrors_(ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper), named); + return this.flatMapValuesCapturingErrorsInternal(ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper), + named); } @Override public KErrorStream flatMapValuesCapturingErrors( final ValueMapperWithKey> mapper, final java.util.function.Predicate errorFilter, final Named named) { - return this.flatMapValuesCapturingErrors_( + return this.flatMapValuesCapturingErrorsInternal( ErrorCapturingFlatValueMapperWithKey.captureErrors(mapper, errorFilter), named); } @@ -872,7 +879,8 @@ public ImprovedKStream process( public KErrorStream processCapturingErrors( final ProcessorSupplier processorSupplier, final String... stateStoreNames) { - return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier), stateStoreNames); + return this.processCapturingErrorsInternal(ErrorCapturingProcessor.captureErrors(processorSupplier), + stateStoreNames); } @Override @@ -880,7 +888,8 @@ public KErrorStream processCapturingErrors( final ProcessorSupplier processorSupplier, final java.util.function.Predicate errorFilter, final String... stateStoreNames) { - return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier, errorFilter), + return this.processCapturingErrorsInternal( + ErrorCapturingProcessor.captureErrors(processorSupplier, errorFilter), stateStoreNames); } @@ -895,7 +904,7 @@ public ImprovedKStream process( public KErrorStream processCapturingErrors( final ProcessorSupplier processorSupplier, final Named named, final String... stateStoreNames) { - return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier), named, + return this.processCapturingErrorsInternal(ErrorCapturingProcessor.captureErrors(processorSupplier), named, stateStoreNames); } @@ -904,7 +913,8 @@ public KErrorStream processCapturingErrors( final ProcessorSupplier processorSupplier, final java.util.function.Predicate errorFilter, final Named named, final String... stateStoreNames) { - return this.processCapturingErrors_(ErrorCapturingProcessor.captureErrors(processorSupplier, errorFilter), + return this.processCapturingErrorsInternal( + ErrorCapturingProcessor.captureErrors(processorSupplier, errorFilter), named, stateStoreNames); } @@ -919,7 +929,7 @@ public ImprovedKStream processValues( public KErrorStream processValuesCapturingErrors( final FixedKeyProcessorSupplier processorSupplier, final String... stateStoreNames) { - return this.processValuesCapturingErrors_(ErrorCapturingValueProcessor.captureErrors(processorSupplier), + return this.processValuesCapturingErrorsInternal(ErrorCapturingValueProcessor.captureErrors(processorSupplier), stateStoreNames); } @@ -927,7 +937,7 @@ public KErrorStream processValuesCapturingErrors( public KErrorStream processValuesCapturingErrors( final FixedKeyProcessorSupplier processorSupplier, final java.util.function.Predicate errorFilter, final String... stateStoreNames) { - return this.processValuesCapturingErrors_( + return this.processValuesCapturingErrorsInternal( ErrorCapturingValueProcessor.captureErrors(processorSupplier, errorFilter), stateStoreNames ); @@ -944,7 +954,8 @@ public ImprovedKStream processValues( public KErrorStream processValuesCapturingErrors( final FixedKeyProcessorSupplier processorSupplier, final Named named, final String... stateStoreNames) { - return this.processValuesCapturingErrors_(ErrorCapturingValueProcessor.captureErrors(processorSupplier), named, + return this.processValuesCapturingErrorsInternal(ErrorCapturingValueProcessor.captureErrors(processorSupplier), + named, stateStoreNames); } @@ -953,84 +964,84 @@ public KErrorStream processValuesCapturingErrors( final FixedKeyProcessorSupplier processorSupplier, final java.util.function.Predicate errorFilter, final Named named, final String... stateStoreNames) { - return this.processValuesCapturingErrors_( + return this.processValuesCapturingErrorsInternal( ErrorCapturingValueProcessor.captureErrors(processorSupplier, errorFilter), named, stateStoreNames); } - private KeyValueKErrorStream mapCapturingErrors_( + private KeyValueKErrorStream mapCapturingErrorsInternal( final KeyValueMapper>> mapper) { final ImprovedKStream> map = this.map(mapper); return new KeyValueKErrorStream<>(map); } - private KeyValueKErrorStream mapCapturingErrors_( + private KeyValueKErrorStream mapCapturingErrorsInternal( final KeyValueMapper>> mapper, final Named named) { final ImprovedKStream> map = this.map(mapper, named); return new KeyValueKErrorStream<>(map); } - private ValueKErrorStream mapValuesCapturingErrors_( + private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapper> mapper) { final ImprovedKStream> map = this.mapValues(mapper); return new ValueKErrorStream<>(map); } - private ValueKErrorStream mapValuesCapturingErrors_( + private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapper> mapper, final Named named) { final ImprovedKStream> map = this.mapValues(mapper, named); return new ValueKErrorStream<>(map); } - private ValueKErrorStream mapValuesCapturingErrors_( + private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapperWithKey> mapper) { final ImprovedKStream> map = this.mapValues(mapper); return new ValueKErrorStream<>(map); } - private ValueKErrorStream mapValuesCapturingErrors_( + private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapperWithKey> mapper, final Named named) { final ImprovedKStream> map = this.mapValues(mapper, named); return new ValueKErrorStream<>(map); } - private KeyValueKErrorStream flatMapCapturingErrors_( + private KeyValueKErrorStream flatMapCapturingErrorsInternal( final KeyValueMapper>>> mapper) { final ImprovedKStream> map = this.flatMap(mapper); return new KeyValueKErrorStream<>(map); } - private KeyValueKErrorStream flatMapCapturingErrors_( + private KeyValueKErrorStream flatMapCapturingErrorsInternal( final KeyValueMapper>>> mapper, final Named named) { final ImprovedKStream> map = this.flatMap(mapper, named); return new KeyValueKErrorStream<>(map); } - private ValueKErrorStream flatMapValuesCapturingErrors_( + private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapper>> mapper) { final ImprovedKStream> map = this.flatMapValues(mapper); return new ValueKErrorStream<>(map); } - private ValueKErrorStream flatMapValuesCapturingErrors_( + private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapper>> mapper, final Named named) { final ImprovedKStream> map = this.flatMapValues(mapper, named); return new ValueKErrorStream<>(map); } - private ValueKErrorStream flatMapValuesCapturingErrors_( + private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapperWithKey>> mapper) { final ImprovedKStream> map = this.flatMapValues(mapper); return new ValueKErrorStream<>(map); } - private ValueKErrorStream flatMapValuesCapturingErrors_( + private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapperWithKey>> mapper, final Named named) { final ImprovedKStream> map = this.flatMapValues(mapper, named); return new ValueKErrorStream<>(map); } - private KeyValueKErrorStream processCapturingErrors_( + private KeyValueKErrorStream processCapturingErrorsInternal( final ProcessorSupplier> processorSupplier, final String... stateStoreNames) { final ImprovedKStream> map = @@ -1038,7 +1049,7 @@ private KeyValueKErrorStream processCapturingErro return new KeyValueKErrorStream<>(map); } - private KeyValueKErrorStream processCapturingErrors_( + private KeyValueKErrorStream processCapturingErrorsInternal( final ProcessorSupplier> processorSupplier, final Named named, final String... stateStoreNames) { final ImprovedKStream> map = @@ -1046,14 +1057,14 @@ private KeyValueKErrorStream processCapturingErro return new KeyValueKErrorStream<>(map); } - private ValueKErrorStream processValuesCapturingErrors_( + private ValueKErrorStream processValuesCapturingErrorsInternal( final FixedKeyProcessorSupplier> processorSupplier, final String... stateStoreNames) { final ImprovedKStream> map = this.processValues(processorSupplier, stateStoreNames); return new ValueKErrorStream<>(map); } - private ValueKErrorStream processValuesCapturingErrors_( + private ValueKErrorStream processValuesCapturingErrorsInternal( final FixedKeyProcessorSupplier> processorSupplier, final Named named, final String... stateStoreNames) { final ImprovedKStream> map = diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index f746bccb..46041222 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -24,11 +24,26 @@ package com.bakdata.kafka; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import io.vavr.collection.List; import java.util.Map; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.STRICT_STUBS) class ImprovedKStreamTest { @Test @@ -204,4 +219,250 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.stop(); } + + @Test + void shouldMapCapturingErrors() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValuesCapturingErrors() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValuesWithKeyCapturingErrors() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFlatMapCapturingErrors() { + final KeyValueMapper>> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.flatMapCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFlatMapValuesCapturingErrors() { + final ValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn(List.of("success")).when(mapper).apply("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.flatMapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFlatMapValuesWithKeyCapturingErrors() { + final ValueMapperWithKey> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of("success")).when(mapper).apply("foo", "baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.flatMapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } } From bec9805735d1b5d7806c2e7ee44f230f4843b936 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 11:17:58 +0100 Subject: [PATCH 17/72] Add tests --- .../bakdata/kafka/ImprovedKStreamTest.java | 211 +++++++++++++----- .../bakdata/kafka/TopologyBuilderTest.java | 74 +++--- 2 files changed, 177 insertions(+), 108 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index 46041222..29bc5b06 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -36,6 +36,14 @@ import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.processor.api.FixedKeyProcessor; +import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; +import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; +import org.apache.kafka.streams.processor.api.FixedKeyRecord; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.ProcessorContext; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.processor.api.Record; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -46,6 +54,15 @@ @MockitoSettings(strictness = Strictness.STRICT_STUBS) class ImprovedKStreamTest { + static TestTopology startApp(final StreamsApp app, final StreamsTopicConfig topicConfig) { + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig)); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + return topology; + } + @Test void shouldWriteToOutput() { final StreamsApp app = new SimpleApp() { @@ -55,13 +72,10 @@ public void buildTopology(final TopologyBuilder builder) { input.toOutputTopic(); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .outputTopic("output") - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); topology.input().add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -83,13 +97,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .outputTopic("output") - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); topology.input() .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) @@ -113,13 +124,10 @@ public void buildTopology(final TopologyBuilder builder) { input.toOutputTopic("label"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .labeledOutputTopics(Map.of("label", "output")) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .labeledOutputTopics(Map.of("label", "output")) + .build()); topology.input().add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -141,13 +149,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .labeledOutputTopics(Map.of("label", "output")) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .labeledOutputTopics(Map.of("label", "output")) + .build()); topology.input() .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) @@ -171,13 +176,10 @@ public void buildTopology(final TopologyBuilder builder) { input.toErrorTopic(); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .errorTopic("error") - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .errorTopic("error") + .build()); topology.input().add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -199,13 +201,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .errorTopic("error") - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .errorTopic("error") + .build()); topology.input() .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) @@ -237,11 +236,8 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder().build()); topology.input().add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); @@ -278,11 +274,8 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder().build()); topology.input().add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); @@ -319,11 +312,8 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder().build()); topology.input().add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); @@ -360,11 +350,8 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder().build()); topology.input().add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); @@ -401,11 +388,8 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder().build()); topology.input().add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); @@ -442,11 +426,118 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder().build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldProcessCapturingErrors() { + final ProcessorSupplier processor = () -> new Processor<>() { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + @Override + public void process(final Record inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.context.forward(inputRecord.withKey("success_key").withValue("success_value")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.processCapturingErrors(processor); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldProcessValuesCapturingErrors() { + final FixedKeyProcessorSupplier processor = () -> new FixedKeyProcessor<>() { + private FixedKeyProcessorContext context = null; + + @Override + public void init(final FixedKeyProcessorContext context) { + this.context = context; + } + + @Override + public void process(final FixedKeyRecord inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.context.forward(inputRecord.withValue("success")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final KErrorStream processed = + input.processValuesCapturingErrors(processor); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); topology.input().add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index fec40988..99db1a3d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -24,6 +24,8 @@ package com.bakdata.kafka; +import static com.bakdata.kafka.ImprovedKStreamTest.startApp; + import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.util.List; import java.util.Map; @@ -42,13 +44,10 @@ public void buildTopology(final TopologyBuilder builder) { input.to("output"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .inputTopics(List.of("input")) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .build()); topology.input().add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -70,13 +69,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .inputTopics(List.of("input")) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .build()); topology.input() .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) @@ -100,13 +96,10 @@ public void buildTopology(final TopologyBuilder builder) { input.to("output"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .labeledInputTopics(Map.of("label", List.of("input"))) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .labeledInputTopics(Map.of("label", List.of("input"))) + .build()); topology.input().add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -128,13 +121,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .labeledInputTopics(Map.of("label", List.of("input"))) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .labeledInputTopics(Map.of("label", List.of("input"))) + .build()); topology.input() .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) @@ -158,13 +148,10 @@ public void buildTopology(final TopologyBuilder builder) { input.to("output"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .inputPattern(Pattern.compile("input\\d+")) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .inputPattern(Pattern.compile("input\\d+")) + .build()); topology.input("input1").add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -186,13 +173,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .inputPattern(Pattern.compile("input\\d+")) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .inputPattern(Pattern.compile("input\\d+")) + .build()); topology.input("input1") .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) @@ -216,13 +200,10 @@ public void buildTopology(final TopologyBuilder builder) { input.to("output"); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) + .build()); topology.input("input1").add("foo", "bar"); topology.streamOutput() .expectNextRecord() @@ -244,13 +225,10 @@ public void buildTopology(final TopologyBuilder builder) { Preconfigured.create(Serdes.Long()))); } }; - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(StreamsTopicConfig.builder() - .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) - .build())); final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); + startApp(app, StreamsTopicConfig.builder() + .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) + .build()); topology.input("input1") .withKeySerde(Serdes.Long()) .withValueSerde(Serdes.Long()) From 21b013c0c82b87bd0d2e22f5f8baa01679e68fce Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 11:53:07 +0100 Subject: [PATCH 18/72] Add ConfiguredStreamJoined --- .../bakdata/kafka/ConfiguredMaterialized.java | 4 +- .../bakdata/kafka/ConfiguredStreamJoined.java | 161 ++++++++++++++++++ .../com/bakdata/kafka/ImprovedKStream.java | 24 +++ .../bakdata/kafka/ImprovedKStreamImpl.java | 42 +++++ .../bakdata/kafka/ImprovedKStreamTest.java | 62 +++++++ 5 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java index b246b102..7214768e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java @@ -128,7 +128,9 @@ public static ConfiguredMaterialized as( } Materialized configure(final Configurator configurator) { - final Materialized materialized = Materialized.as(this.storeName) + final Materialized materialized = + this.storeName == null ? Materialized.with(configurator.configureForKeys(this.keySerde), + configurator.configureForValues(this.valueSerde)) : Materialized.as(this.storeName) .withKeySerde(configurator.configureForKeys(this.keySerde)) .withValueSerde(configurator.configureForValues(this.valueSerde)); if (this.retention != null) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java new file mode 100644 index 00000000..9a877055 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java @@ -0,0 +1,161 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.HashMap; +import java.util.Map; +import lombok.AccessLevel; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.apache.kafka.streams.state.DslStoreSuppliers; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link StreamJoined} using {@link Configurator} + * @param type of keys + * @param this value type + * @param other value type + * @see StreamJoined + */ +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public final class ConfiguredStreamJoined { + + @With + private final @NonNull Preconfigured> keySerde; + @With + private final @NonNull Preconfigured> valueSerde; + @With + private final @NonNull Preconfigured> otherValueSerde; + @With + private final DslStoreSuppliers dslStoreSuppliers; + @With + private final String name; + @With + private final String storeName; + private final Map topicConfig; + private final boolean loggingEnabled; + + /** + * Create an instance of {@code ConfiguredStreamJoined} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredStreamJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredStreamJoined keySerde( + final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code ConfiguredStreamJoined} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredStreamJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredStreamJoined valueSerde( + final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code ConfiguredStreamJoined} with provided other value serde + * @param valueSerde Serde to use for other values + * @return a new instance of {@code ConfiguredStreamJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredStreamJoined otherValueSerde( + final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), valueSerde); + } + + /** + * Create an instance of {@code ConfiguredStreamJoined} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredStreamJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredStreamJoined with( + final Preconfigured> keySerde, + final Preconfigured> valueSerde, + final Preconfigured> otherValueSerde) { + return new ConfiguredStreamJoined<>(keySerde, valueSerde, otherValueSerde, null, null, null, new HashMap<>(), + true); + } + + /** + * Create an instance of {@code ConfiguredStreamJoined} with provided store name + * @param storeName the store name to be used + * @return a new instance of {@code ConfiguredStreamJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredStreamJoined as(final String storeName) { + return new ConfiguredStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde(), + null, null, storeName, new HashMap<>(), true); + } + + /** + * Create an instance of {@code ConfiguredStreamJoined} with provided store suppliers + * @param storeSuppliers the store suppliers to be used + * @return a new instance of {@code ConfiguredStreamJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredStreamJoined as( + final DslStoreSuppliers storeSuppliers) { + return new ConfiguredStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde(), storeSuppliers, + null, null, new HashMap<>(), true); + } + + StreamJoined configure(final Configurator configurator) { + final StreamJoined streamJoined = StreamJoined.as(this.storeName) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) + .withOtherValueSerde(configurator.configureForValues(this.otherValueSerde)) + .withName(this.name) + .withDslStoreSuppliers(this.dslStoreSuppliers); + if (this.loggingEnabled) { + return streamJoined.withLoggingEnabled(this.topicConfig); + } else { + return streamJoined.withLoggingDisabled(); + } + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 82a27fe5..a8e97da6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -308,11 +308,19 @@ ImprovedKStream join(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); + ImprovedKStream join(KStream otherStream, + ValueJoiner joiner, + JoinWindows windows, ConfiguredStreamJoined streamJoined); + @Override ImprovedKStream join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); + ImprovedKStream join(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows, + ConfiguredStreamJoined streamJoined); + @Override ImprovedKStream leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows); @@ -326,11 +334,19 @@ ImprovedKStream leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); + ImprovedKStream leftJoin(KStream otherStream, + ValueJoiner joiner, JoinWindows windows, + ConfiguredStreamJoined streamJoined); + @Override ImprovedKStream leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); + ImprovedKStream leftJoin(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows, + ConfiguredStreamJoined streamJoined); + @Override ImprovedKStream outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows); @@ -344,11 +360,19 @@ ImprovedKStream outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); + ImprovedKStream outerJoin(KStream otherStream, + ValueJoiner joiner, JoinWindows windows, + ConfiguredStreamJoined streamJoined); + @Override ImprovedKStream outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); + ImprovedKStream outerJoin(KStream otherStream, + ValueJoinerWithKey joiner, JoinWindows windows, + ConfiguredStreamJoined streamJoined); + @Override ImprovedKStream join(KTable table, ValueJoiner joiner); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 47f767b9..535f153a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -581,6 +581,13 @@ public ImprovedKStream join(final KStream otherStream, return this.context.wrap(this.wrapped.join(other, joiner, windows, streamJoined)); } + @Override + public ImprovedKStream join(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows, + final ConfiguredStreamJoined streamJoined) { + return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, @@ -589,6 +596,13 @@ public ImprovedKStream join(final KStream otherStream, return this.context.wrap(this.wrapped.join(other, joiner, windows, streamJoined)); } + @Override + public ImprovedKStream join(final KStream otherStream, + final ValueJoinerWithKey joiner, final JoinWindows windows, + final ConfiguredStreamJoined streamJoined) { + return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { @@ -612,6 +626,13 @@ public ImprovedKStream leftJoin(final KStream otherStream return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows, streamJoined)); } + @Override + public ImprovedKStream leftJoin(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows, + final ConfiguredStreamJoined streamJoined) { + return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, @@ -620,6 +641,13 @@ public ImprovedKStream leftJoin(final KStream otherStream return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows, streamJoined)); } + @Override + public ImprovedKStream leftJoin(final KStream otherStream, + final ValueJoinerWithKey joiner, final JoinWindows windows, + final ConfiguredStreamJoined streamJoined) { + return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { @@ -643,6 +671,13 @@ public ImprovedKStream outerJoin(final KStream otherStrea return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows, streamJoined)); } + @Override + public ImprovedKStream outerJoin(final KStream otherStream, + final ValueJoiner joiner, final JoinWindows windows, + final ConfiguredStreamJoined streamJoined) { + return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, @@ -651,6 +686,13 @@ public ImprovedKStream outerJoin(final KStream otherStrea return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows, streamJoined)); } + @Override + public ImprovedKStream outerJoin(final KStream otherStream, + final ValueJoinerWithKey joiner, final JoinWindows windows, + final ConfiguredStreamJoined streamJoined) { + return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream join(final KTable table, final ValueJoiner joiner) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index 29bc5b06..0c3f93f0 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -556,4 +556,66 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.stop(); } + + @Test + void shouldRepartition() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream repartitioned = input.repartition( + ConfiguredRepartitioned.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + repartitioned.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldConvertToTable() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKTable table = input.toTable( + ConfiguredMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + table.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } } From 0f401eb73f269e82981f5d8526425d874efbd420 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 13:45:08 +0100 Subject: [PATCH 19/72] Add tests --- .../bakdata/kafka/ImprovedKStreamTest.java | 326 +++++++++++++++++- 1 file changed, 310 insertions(+), 16 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index 0c3f93f0..c27608e9 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -30,9 +30,11 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import io.vavr.collection.List; +import java.time.Duration; import java.util.Map; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; @@ -449,13 +451,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldProcessCapturingErrors() { - final ProcessorSupplier processor = () -> new Processor<>() { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { @Override public void process(final Record inputRecord) { @@ -463,7 +459,7 @@ public void process(final Record inputRecord) { throw new RuntimeException("Cannot process"); } if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { - this.context.forward(inputRecord.withKey("success_key").withValue("success_value")); + this.forward(inputRecord.withKey("success_key").withValue("success_value")); return; } throw new UnsupportedOperationException(); @@ -504,13 +500,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldProcessValuesCapturingErrors() { - final FixedKeyProcessorSupplier processor = () -> new FixedKeyProcessor<>() { - private FixedKeyProcessorContext context = null; - - @Override - public void init(final FixedKeyProcessorContext context) { - this.context = context; - } + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { @Override public void process(final FixedKeyRecord inputRecord) { @@ -518,7 +508,7 @@ public void process(final FixedKeyRecord inputRecord) { throw new RuntimeException("Cannot process"); } if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { - this.context.forward(inputRecord.withValue("success")); + this.forward(inputRecord.withValue("success")); return; } throw new UnsupportedOperationException(); @@ -618,4 +608,308 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.stop(); } + + @Test + void shouldJoin() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream otherInput = builder.stream("other_input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream joined = input.join(otherInput, + Long::sum, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); + joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldJoinWithKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream otherInput = builder.stream("other_input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream joined = input.join(otherInput, + (k, v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); + joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldLeftJoin() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream otherInput = builder.stream("other_input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); + joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldLeftJoinWithKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream otherInput = builder.stream("other_input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream joined = input.leftJoin(otherInput, + (k, v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); + joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldOuterJoin() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream otherInput = builder.stream("other_input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream joined = input.outerJoin(otherInput, + (v1, v2) -> v1 == null ? v2 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); + joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(3L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldOuterJoinWithKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream otherInput = builder.stream("other_input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKStream joined = input.outerJoin(otherInput, + (k, v1, v2) -> v1 == null ? v2 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); + joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(3L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + private abstract static class SimpleProcessor implements Processor { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + protected void forward(final Record outputRecord) { + this.context.forward(outputRecord); + } + + } + + private abstract static class SimpleFixedKeyProcessor implements FixedKeyProcessor { + private FixedKeyProcessorContext context = null; + + @Override + public void init(final FixedKeyProcessorContext context) { + this.context = context; + } + + protected void forward(final FixedKeyRecord outputRecord) { + this.context.forward(outputRecord); + } + + } } From e22d8b28b17e6438533dad0f4aa4d5d5f215e378 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 14:11:35 +0100 Subject: [PATCH 20/72] Add tests --- .../com/bakdata/kafka/ConfiguredGrouped.java | 102 +++++++++++ .../com/bakdata/kafka/ImprovedKStream.java | 5 + .../bakdata/kafka/ImprovedKStreamImpl.java | 11 ++ .../com/bakdata/kafka/ImprovedKTable.java | 3 + .../com/bakdata/kafka/ImprovedKTableImpl.java | 6 + .../kafka/ImprovedKGroupedStreamTest.java | 166 ++++++++++++++++++ .../bakdata/kafka/ImprovedKStreamTest.java | 150 ++++++++++++++++ 7 files changed, 443 insertions(+) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java new file mode 100644 index 00000000..8072cd08 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java @@ -0,0 +1,102 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import lombok.AccessLevel; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Grouped; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Grouped} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Grouped + */ +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public final class ConfiguredGrouped { + + @With + private final @NonNull Preconfigured> keySerde; + @With + private final @NonNull Preconfigured> valueSerde; + @With + private final String name; + + /** + * Create an instance of {@code ConfiguredGrouped} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredGrouped} + * @param type of keys + * @param type of values + */ + public static ConfiguredGrouped keySerde(final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code ConfiguredGrouped} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredGrouped} + * @param type of keys + * @param type of values + */ + public static ConfiguredGrouped valueSerde(final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde); + } + + /** + * Create an instance of {@code ConfiguredGrouped} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredGrouped} + * @param type of keys + * @param type of values + */ + public static ConfiguredGrouped with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ConfiguredGrouped<>(keySerde, valueSerde, null); + } + + /** + * Create an instance of {@code ConfiguredGrouped} with provided name + * @param name the name used for a repartition topic if required + * @return a new instance of {@code ConfiguredGrouped} + * @param type of keys + * @param type of values + */ + public static ConfiguredGrouped as(final String name) { + return new ConfiguredGrouped<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), name); + } + + Grouped configure(final Configurator configurator) { + return Grouped.as(this.name) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)); + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index a8e97da6..50957b70 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -288,12 +288,17 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, Grouped grouped); + ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, + ConfiguredGrouped grouped); + @Override ImprovedKGroupedStream groupByKey(); @Override ImprovedKGroupedStream groupByKey(Grouped grouped); + ImprovedKGroupedStream groupByKey(ConfiguredGrouped grouped); + @Override ImprovedKStream join(KStream otherStream, ValueJoiner joiner, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 535f153a..e36031a7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -548,6 +548,12 @@ public ImprovedKGroupedStream groupBy(final KeyValueMapper ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector, + final ConfiguredGrouped grouped) { + return this.groupBy(keySelector, grouped.configure(this.context.getConfigurator())); + } + @Override public ImprovedKGroupedStream groupByKey() { return this.context.wrap(this.wrapped.groupByKey()); @@ -558,6 +564,11 @@ public ImprovedKGroupedStream groupByKey(final Grouped grouped) { return this.context.wrap(this.wrapped.groupByKey(grouped)); } + @Override + public ImprovedKGroupedStream groupByKey(final ConfiguredGrouped grouped) { + return this.groupByKey(grouped.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index 8440e7d4..dc79d924 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -180,6 +180,9 @@ ImprovedKTable transformValues( ImprovedKGroupedTable groupBy(KeyValueMapper> selector, Grouped grouped); + ImprovedKGroupedTable groupBy(KeyValueMapper> selector, + ConfiguredGrouped grouped); + @Override ImprovedKTable join(KTable other, ValueJoiner joiner); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java index efe30688..df88fba0 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -276,6 +276,12 @@ public ImprovedKGroupedTable groupBy( return this.context.wrap(this.wrapped.groupBy(selector, grouped)); } + @Override + public ImprovedKGroupedTable groupBy( + final KeyValueMapper> selector, final ConfiguredGrouped grouped) { + return this.groupBy(selector, grouped.configure(this.context.getConfigurator())); + } + @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java new file mode 100644 index 00000000..d7f4487b --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java @@ -0,0 +1,166 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static com.bakdata.kafka.ImprovedKStreamTest.startApp; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import org.apache.kafka.common.serialization.Serdes; +import org.junit.jupiter.api.Test; + +class ImprovedKGroupedStreamTest { + + @Test + void shouldReduce() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKGroupedStream grouped = input.groupByKey(); + final ImprovedKTable reduced = grouped.reduce((value1, value2) -> value1 + value2); + reduced.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldReduceUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKGroupedStream grouped = input.groupByKey( + ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKTable reduced = grouped.reduce(Long::sum, + ConfiguredMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + reduced.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldAggregate() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKGroupedStream grouped = input.groupByKey(); + final ImprovedKTable aggregated = + grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value); + aggregated.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldAggregateUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKGroupedStream grouped = input.groupByKey( + ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKTable aggregated = + grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, + ConfiguredMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + aggregated.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index c27608e9..b9318103 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -549,6 +549,28 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldRepartition() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream repartitioned = input.repartition(); + repartitioned.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldRepartitionUsingRepartitioned() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -885,6 +907,134 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldGroupByKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKGroupedStream grouped = input.groupByKey(); + final ImprovedKTable count = grouped.count(); + count.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldGroupByKeyUsingGrouped() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKGroupedStream grouped = input.groupByKey( + ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKTable count = grouped.count(); + count.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldGroupBy() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKGroupedStream grouped = input.groupBy((k, v) -> v); + final ImprovedKTable count = grouped.count(); + count.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldGroupByUsingGrouped() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input", + ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKGroupedStream grouped = input.groupBy((k, v) -> v, + ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final ImprovedKTable count = grouped.count(); + count.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(2L) + .hasValue(1L) + .expectNextRecord() + .hasKey(2L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; From e529a2af7d1735be419376e5f8c24603942202ee Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 14:30:52 +0100 Subject: [PATCH 21/72] Add ConfiguredJoined --- .../com/bakdata/kafka/ConfiguredJoined.java | 130 ++++++++++++++++++ .../bakdata/kafka/ConfiguredStreamJoined.java | 1 + .../kafka/ImprovedBranchedKStreamImpl.java | 2 +- .../com/bakdata/kafka/ImprovedKStream.java | 15 ++ .../bakdata/kafka/ImprovedKStreamImpl.java | 26 ++++ 5 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java new file mode 100644 index 00000000..134bb568 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java @@ -0,0 +1,130 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.time.Duration; +import lombok.AccessLevel; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Joined; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Joined} using {@link Configurator} + * @param type of keys + * @param this value type + * @param other value type + * @see Joined + */ +@With +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public final class ConfiguredJoined { + + private final @NonNull Preconfigured> keySerde; + private final @NonNull Preconfigured> valueSerde; + private final @NonNull Preconfigured> otherValueSerde; + private final String name; + private final Duration gracePeriod; + + /** + * Create an instance of {@code ConfiguredJoined} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConfiguredJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredJoined keySerde( + final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code ConfiguredJoined} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConfiguredJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredJoined valueSerde( + final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code ConfiguredJoined} with provided other value serde + * @param valueSerde Serde to use for other values + * @return a new instance of {@code ConfiguredJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredJoined otherValueSerde( + final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), valueSerde); + } + + /** + * Create an instance of {@code ConfiguredJoined} with provided key and value serde + * @param keySerde Serde to use for keys + * @param valueSerde Serde to use for values + * @param otherValueSerde Serde to use for other values + * @return a new instance of {@code ConfiguredJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredJoined with( + final Preconfigured> keySerde, + final Preconfigured> valueSerde, + final Preconfigured> otherValueSerde) { + return new ConfiguredJoined<>(keySerde, valueSerde, otherValueSerde, null, null); + } + + /** + * Create an instance of {@code ConfiguredJoined} with provided store name + * @param name the name used as the base for naming components of the join including any repartition topics + * @return a new instance of {@code ConfiguredJoined} + * @param type of keys + * @param this value type + * @param other value type + */ + public static ConfiguredJoined as(final String name) { + return new ConfiguredJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde(), name, null); + } + + Joined configure(final Configurator configurator) { + return Joined.as(this.name) + .withKeySerde(configurator.configureForKeys(this.keySerde)) + .withValueSerde(configurator.configureForValues(this.valueSerde)) + .withOtherValueSerde(configurator.configureForValues(this.otherValueSerde)) + .withName(this.name) + .withGracePeriod(this.gracePeriod); + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java index 9a877055..bc977c55 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java @@ -102,6 +102,7 @@ public static ConfiguredStreamJoined otherValueSerde( * Create an instance of {@code ConfiguredStreamJoined} with provided key and value serde * @param keySerde Serde to use for keys * @param valueSerde Serde to use for values + * @param otherValueSerde Serde to use for other values * @return a new instance of {@code ConfiguredStreamJoined} * @param type of keys * @param this value type diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java index bcd30805..2f390a03 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java @@ -35,7 +35,7 @@ import org.apache.kafka.streams.kstream.Predicate; @RequiredArgsConstructor -public class ImprovedBranchedKStreamImpl implements ImprovedBranchedKStream { +class ImprovedBranchedKStreamImpl implements ImprovedBranchedKStream { private final @NonNull BranchedKStream wrapped; private final @NonNull StreamsContext context; diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 50957b70..9a2b52c4 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -389,10 +389,17 @@ ImprovedKStream join(KTable table, ImprovedKStream join(KTable table, ValueJoiner joiner, Joined joined); + ImprovedKStream join(KTable table, ValueJoiner joiner, + ConfiguredJoined joined); + @Override ImprovedKStream join(KTable table, ValueJoinerWithKey joiner, Joined joined); + ImprovedKStream join(KTable table, + ValueJoinerWithKey joiner, + ConfiguredJoined joined); + @Override ImprovedKStream leftJoin(KTable table, ValueJoiner joiner); @@ -406,10 +413,18 @@ ImprovedKStream leftJoin(KTable table, ValueJoiner joiner, Joined joined); + ImprovedKStream leftJoin(KTable table, + ValueJoiner joiner, + ConfiguredJoined joined); + @Override ImprovedKStream leftJoin(KTable table, ValueJoinerWithKey joiner, Joined joined); + ImprovedKStream leftJoin(KTable table, + ValueJoinerWithKey joiner, + ConfiguredJoined joined); + @Override ImprovedKStream join(GlobalKTable globalTable, KeyValueMapper keySelector, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index e36031a7..478e0bd6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -725,6 +725,12 @@ public ImprovedKStream join(final KTable table, return this.context.wrap(this.wrapped.join(other, joiner, joined)); } + @Override + public ImprovedKStream join(final KTable table, + final ValueJoiner joiner, final ConfiguredJoined joined) { + return this.join(table, joiner, joined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream join(final KTable table, final ValueJoinerWithKey joiner, @@ -733,6 +739,13 @@ public ImprovedKStream join(final KTable table, return this.context.wrap(this.wrapped.join(other, joiner, joined)); } + @Override + public ImprovedKStream join(final KTable table, + final ValueJoinerWithKey joiner, + final ConfiguredJoined joined) { + return this.join(table, joiner, joined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoiner joiner) { @@ -754,6 +767,12 @@ public ImprovedKStream leftJoin(final KTable table, return this.context.wrap(this.wrapped.leftJoin(other, joiner, joined)); } + @Override + public ImprovedKStream leftJoin(final KTable table, + final ValueJoiner joiner, final ConfiguredJoined joined) { + return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoinerWithKey joiner, @@ -762,6 +781,13 @@ public ImprovedKStream leftJoin(final KTable table, return this.context.wrap(this.wrapped.leftJoin(other, joiner, joined)); } + @Override + public ImprovedKStream leftJoin(final KTable table, + final ValueJoinerWithKey joiner, + final ConfiguredJoined joined) { + return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); + } + @Override public ImprovedKStream join(final GlobalKTable globalTable, final KeyValueMapper keySelector, From 000360d903016dba8abbc584b6e7d8405c26a8dc Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 14:51:59 +0100 Subject: [PATCH 22/72] Add docs --- .../com/bakdata/kafka/ImprovedKStream.java | 85 +++++++++++++++++++ .../com/bakdata/kafka/ImprovedKTable.java | 69 +++++++++++++++ 2 files changed, 154 insertions(+) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 9a2b52c4..64b8ef25 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -241,10 +241,32 @@ KErrorStream flatMapValuesCapturingErrors( @Override ImprovedKStream repartition(Repartitioned repartitioned); + /** + * Materialize this stream to an auto-generated repartition topic and create a new KStream from the + * auto-generated topic + * @param repartitioned the {@code Repartitioned} instance used to specify {@code Serdes}, {@code StreamPartitioner} + * which determines how records are distributed among partitions of the topic, part of the topic name, and number + * of partitions + * for a repartition topic + * @return a {@code KStream} that contains the exact same repartitioned records as this {@code KStream} + * @see #repartition(Repartitioned) + */ ImprovedKStream repartition(ConfiguredRepartitioned repartitioned); + /** + * Materialize this stream to a topic using the provided {@code ConfiguredProduced} instance + * @param topic the topic name + * @param produced the options to use when producing to the topic + * @see #to(String, Produced) + */ void to(String topic, ConfiguredProduced produced); + /** + * Dynamically materialize this stream to topics using the provided {@code ConfiguredProduced} instance + * @param topicExtractor the extractor to determine the name of the Kafka topic to write to for each record + * @param produced the options to use when producing to the topic + * @see #to(TopicNameExtractor, Produced) + */ void to(TopicNameExtractor topicExtractor, ConfiguredProduced produced); void toOutputTopic(); @@ -274,11 +296,28 @@ KErrorStream flatMapValuesCapturingErrors( @Override ImprovedKTable toTable(Materialized> materialized); + /** + * Convert this stream to a {@code KTable} + * @param materialized an instance of {@code ConfiguredMaterialized} used to describe how the state store of the + * resulting table + * should be materialized + * @return a {@code KTable} that contains the same records as this {@code KStream} + * @see #toTable(Materialized) + */ ImprovedKTable toTable(ConfiguredMaterialized> materialized); @Override ImprovedKTable toTable(Named named, Materialized> materialized); + /** + * Convert this stream to a {@code KTable} + * @param named a {@code Named} config used to name the processor in the topology + * @param materialized an instance of {@code ConfiguredMaterialized} used to describe how the state store of the + * resulting table + * should be materialized + * @return a {@code KTable} that contains the same records as this {@code KStream} + * @see #toTable(Named, Materialized) + */ ImprovedKTable toTable(Named named, ConfiguredMaterialized> materialized); @Override @@ -288,6 +327,15 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, Grouped grouped); + /** + * Group the records of this {@code KStream} on a new key that is selected using the provided {@code KeyValueMapper} + * @param keySelector a {@code KeyValueMapper} that computes a new key for grouping + * @param grouped the {@code ConfiguredGrouped} instance used to specify {@code Serdes} and part of the name for a + * repartition topic if repartitioning is required + * @return a {@code KGroupedStream} that contains the grouped records of the original {@code KStream} + * @param the key type of the result {@code KGroupedStream} + * @see #groupBy(KeyValueMapper, Grouped) + */ ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, ConfiguredGrouped grouped); @@ -297,6 +345,13 @@ ImprovedKGroupedStream groupBy(KeyValueMapper groupByKey(Grouped grouped); + /** + * Group the records by their current key into a {@code KGroupedStream} while preserving the original values + * @param grouped the {@code ConfiguredGrouped} instance used to specify {@code Serdes} and part of the name for a + * repartition topic if repartitioning is required + * @return a {@code KGroupedStream} that contains the grouped records of the original {@code KStream} + * @see #groupByKey(Grouped) + */ ImprovedKGroupedStream groupByKey(ConfiguredGrouped grouped); @Override @@ -313,6 +368,9 @@ ImprovedKStream join(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); + /** + * @see #join(KStream, ValueJoiner, JoinWindows, StreamJoined) + */ ImprovedKStream join(KStream otherStream, ValueJoiner joiner, JoinWindows windows, ConfiguredStreamJoined streamJoined); @@ -322,6 +380,9 @@ ImprovedKStream join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); + /** + * @see #join(KStream, ValueJoinerWithKey, JoinWindows, StreamJoined) + */ ImprovedKStream join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, ConfiguredStreamJoined streamJoined); @@ -339,6 +400,9 @@ ImprovedKStream leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); + /** + * @see #leftJoin(KStream, ValueJoiner, JoinWindows, StreamJoined) + */ ImprovedKStream leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, ConfiguredStreamJoined streamJoined); @@ -348,6 +412,9 @@ ImprovedKStream leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); + /** + * @see #leftJoin(KStream, ValueJoinerWithKey, JoinWindows, StreamJoined) + */ ImprovedKStream leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, ConfiguredStreamJoined streamJoined); @@ -365,6 +432,9 @@ ImprovedKStream outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); + /** + * @see #outerJoin(KStream, ValueJoiner, JoinWindows, StreamJoined) + */ ImprovedKStream outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, ConfiguredStreamJoined streamJoined); @@ -374,6 +444,9 @@ ImprovedKStream outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); + /** + * @see #outerJoin(KStream, ValueJoinerWithKey, JoinWindows, StreamJoined) + */ ImprovedKStream outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, ConfiguredStreamJoined streamJoined); @@ -389,6 +462,9 @@ ImprovedKStream join(KTable table, ImprovedKStream join(KTable table, ValueJoiner joiner, Joined joined); + /** + * @see #join(KTable, ValueJoiner, Joined) + */ ImprovedKStream join(KTable table, ValueJoiner joiner, ConfiguredJoined joined); @@ -396,6 +472,9 @@ ImprovedKStream join(KTable table, ValueJoiner ImprovedKStream join(KTable table, ValueJoinerWithKey joiner, Joined joined); + /** + * @see #join(KTable, ValueJoinerWithKey, Joined) + */ ImprovedKStream join(KTable table, ValueJoinerWithKey joiner, ConfiguredJoined joined); @@ -413,6 +492,9 @@ ImprovedKStream leftJoin(KTable table, ValueJoiner joiner, Joined joined); + /** + * @see #leftJoin(KTable, ValueJoiner, Joined) + */ ImprovedKStream leftJoin(KTable table, ValueJoiner joiner, ConfiguredJoined joined); @@ -421,6 +503,9 @@ ImprovedKStream leftJoin(KTable table, ImprovedKStream leftJoin(KTable table, ValueJoinerWithKey joiner, Joined joined); + /** + * @see #leftJoin(KTable, ValueJoinerWithKey, Joined) + */ ImprovedKStream leftJoin(KTable table, ValueJoinerWithKey joiner, ConfiguredJoined joined); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index dc79d924..a1280c20 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -59,6 +59,9 @@ public interface ImprovedKTable extends KTable { ImprovedKTable filter(Predicate predicate, Materialized> materialized); + /** + * @see #filter(Predicate, Materialized) + */ ImprovedKTable filter(Predicate predicate, ConfiguredMaterialized> materialized); @@ -66,6 +69,9 @@ ImprovedKTable filter(Predicate predicate, ImprovedKTable filter(Predicate predicate, Named named, Materialized> materialized); + /** + * @see #filter(Predicate, Named, Materialized) + */ ImprovedKTable filter(Predicate predicate, Named named, ConfiguredMaterialized> materialized); @@ -79,6 +85,9 @@ ImprovedKTable filter(Predicate predicate, Named nam ImprovedKTable filterNot(Predicate predicate, Materialized> materialized); + /** + * @see #filterNot(Predicate, Materialized) + */ ImprovedKTable filterNot(Predicate predicate, ConfiguredMaterialized> materialized); @@ -86,6 +95,9 @@ ImprovedKTable filterNot(Predicate predicate, ImprovedKTable filterNot(Predicate predicate, Named named, Materialized> materialized); + /** + * @see #filterNot(Predicate, Named, Materialized) + */ ImprovedKTable filterNot(Predicate predicate, Named named, ConfiguredMaterialized> materialized); @@ -105,6 +117,9 @@ ImprovedKTable filterNot(Predicate predicate, Named ImprovedKTable mapValues(ValueMapper mapper, Materialized> materialized); + /** + * @see #mapValues(ValueMapper, Materialized) + */ ImprovedKTable mapValues(ValueMapper mapper, ConfiguredMaterialized> materialized); @@ -112,6 +127,9 @@ ImprovedKTable mapValues(ValueMapper mapper ImprovedKTable mapValues(ValueMapper mapper, Named named, Materialized> materialized); + /** + * @see #mapValues(ValueMapper, Named, Materialized) + */ ImprovedKTable mapValues(ValueMapper mapper, Named named, ConfiguredMaterialized> materialized); @@ -119,6 +137,9 @@ ImprovedKTable mapValues(ValueMapper mapper ImprovedKTable mapValues(ValueMapperWithKey mapper, Materialized> materialized); + /** + * @see #mapValues(ValueMapperWithKey, Materialized) + */ ImprovedKTable mapValues(ValueMapperWithKey mapper, ConfiguredMaterialized> materialized); @@ -126,6 +147,9 @@ ImprovedKTable mapValues(ValueMapperWithKey ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, Materialized> materialized); + /** + * @see #mapValues(ValueMapperWithKey, Named, Materialized) + */ ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, ConfiguredMaterialized> materialized); @@ -159,6 +183,9 @@ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, Materialized> materialized, String... stateStoreNames); + /** + * @see #transformValues(ValueTransformerWithKeySupplier, Materialized, String...) + */ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, ConfiguredMaterialized> materialized, String... stateStoreNames); @@ -168,6 +195,9 @@ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, Materialized> materialized, Named named, String... stateStoreNames); + /** + * @see #transformValues(ValueTransformerWithKeySupplier, Materialized, Named, String...) + */ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, ConfiguredMaterialized> materialized, Named named, @@ -180,6 +210,9 @@ ImprovedKTable transformValues( ImprovedKGroupedTable groupBy(KeyValueMapper> selector, Grouped grouped); + /** + * @see #groupBy(KeyValueMapper, Grouped) + */ ImprovedKGroupedTable groupBy(KeyValueMapper> selector, ConfiguredGrouped grouped); @@ -194,6 +227,9 @@ ImprovedKTable join(KTable other, ValueJoiner ImprovedKTable join(KTable other, ValueJoiner joiner, Materialized> materialized); + /** + * @see #join(KTable, ValueJoiner, Materialized) + */ ImprovedKTable join(KTable other, ValueJoiner joiner, ConfiguredMaterialized> materialized); @@ -201,6 +237,9 @@ ImprovedKTable join(KTable other, ValueJoiner ImprovedKTable join(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); + /** + * @see #join(KTable, ValueJoiner, Named, Materialized) + */ ImprovedKTable join(KTable other, ValueJoiner joiner, Named named, ConfiguredMaterialized> materialized); @@ -218,6 +257,9 @@ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, Materialized> materialized); + /** + * @see #leftJoin(KTable, ValueJoiner, Materialized) + */ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, ConfiguredMaterialized> materialized); @@ -227,6 +269,9 @@ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); + /** + * @see #leftJoin(KTable, ValueJoiner, Named, Materialized) + */ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, Named named, ConfiguredMaterialized> materialized); @@ -245,6 +290,9 @@ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, Materialized> materialized); + /** + * @see #outerJoin(KTable, ValueJoiner, Materialized) + */ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, ConfiguredMaterialized> materialized); @@ -254,6 +302,9 @@ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); + /** + * @see #outerJoin(KTable, ValueJoiner, Named, Materialized) + */ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, Named named, ConfiguredMaterialized> materialized); @@ -274,6 +325,9 @@ ImprovedKTable join(KTable other, Function fo ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Materialized> materialized); + /** + * @see #join(KTable, Function, ValueJoiner, Materialized) + */ ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, ConfiguredMaterialized> materialized); @@ -281,6 +335,9 @@ ImprovedKTable join(KTable other, Function fo ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); + /** + * @see #join(KTable, Function, ValueJoiner, Named, Materialized) + */ ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, ConfiguredMaterialized> materialized); @@ -290,6 +347,9 @@ ImprovedKTable join(KTable other, Function fo ValueJoiner joiner, TableJoined tableJoined, Materialized> materialized); + /** + * @see #join(KTable, Function, ValueJoiner, TableJoined, Materialized) + */ ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, ConfiguredMaterialized> materialized); @@ -310,6 +370,9 @@ ImprovedKTable leftJoin(KTable other, Function ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Materialized> materialized); + /** + * @see #leftJoin(KTable, Function, ValueJoiner, Materialized) + */ ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, ConfiguredMaterialized> materialized); @@ -317,6 +380,9 @@ ImprovedKTable leftJoin(KTable other, Function ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); + /** + * @see #leftJoin(KTable, Function, ValueJoiner, Named, Materialized) + */ ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, ConfiguredMaterialized> materialized); @@ -326,6 +392,9 @@ ImprovedKTable leftJoin(KTable other, Function joiner, TableJoined tableJoined, Materialized> materialized); + /** + * @see #leftJoin(KTable, Function, ValueJoiner, TableJoined, Materialized) + */ ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, ConfiguredMaterialized> materialized); From efc1a693de2564da2835ce4437b483d3a2f5ce62 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 15:01:01 +0100 Subject: [PATCH 23/72] Add docs --- .../kafka/ImprovedCogroupedKStream.java | 11 ---- .../bakdata/kafka/ImprovedKGroupedStream.java | 35 ------------- .../bakdata/kafka/ImprovedKGroupedTable.java | 45 ---------------- .../com/bakdata/kafka/ImprovedKStream.java | 34 ------------- ...provedSessionWindowedCogroupedKStream.java | 15 ------ .../kafka/ImprovedSessionWindowedKStream.java | 41 --------------- .../ImprovedTimeWindowedCogroupedKStream.java | 13 ----- .../kafka/ImprovedTimeWindowedKStream.java | 43 ---------------- .../com/bakdata/kafka/TopologyBuilder.java | 51 ------------------- 9 files changed, 288 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java index 5a183ac7..504b4f84 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -60,11 +60,6 @@ ImprovedKTable aggregate(Initializer initializer, Materialized> materialized); /** - * Aggregate the values of records in these streams by the grouped key - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key * @see #aggregate(Initializer, Materialized) */ ImprovedKTable aggregate(Initializer initializer, @@ -75,12 +70,6 @@ ImprovedKTable aggregate(Initializer initializer, Named named, Materialized> materialized); /** - * Aggregate the values of records in these streams by the grouped key - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param named name the processors - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key * @see #aggregate(Initializer, Named, Materialized) */ ImprovedKTable aggregate(Initializer initializer, Named named, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index 70d1c72f..bbbba7ae 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -55,10 +55,6 @@ public interface ImprovedKGroupedStream extends KGroupedStream { ImprovedKTable count(Materialized> materialized); /** - * Count the number of records in this stream by the grouped key - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys and {@code Long} values that - * represent the latest (rolling) count (i.e., number of records) for each key * @see #count(Materialized) */ ImprovedKTable count(ConfiguredMaterialized> materialized); @@ -67,11 +63,6 @@ public interface ImprovedKGroupedStream extends KGroupedStream { ImprovedKTable count(Named named, Materialized> materialized); /** - * Count the number of records in this stream by the grouped key - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys and {@code Long} values that - * represent the latest (rolling) count (i.e., number of records) for each key * @see #count(Named, Materialized) */ ImprovedKTable count(Named named, @@ -84,11 +75,6 @@ ImprovedKTable count(Named named, ImprovedKTable reduce(Reducer reducer, Materialized> materialized); /** - * Combine the value of records in this stream by the grouped key - * @param reducer a {@code Reducer} that computes a new aggregate result - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key * @see #reduce(Reducer, Materialized) */ ImprovedKTable reduce(Reducer reducer, @@ -99,12 +85,6 @@ ImprovedKTable reduce(Reducer reducer, Named named, Materialized> materialized); /** - * Combine the value of records in this stream by the grouped key - * @param reducer a {@code Reducer} that computes a new aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key * @see #reduce(Reducer, Named, Materialized) */ ImprovedKTable reduce(Reducer reducer, Named named, @@ -118,13 +98,6 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator> materialized); /** - * Aggregate the values of records in this stream by the grouped key - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@code Aggregator} that computes a new aggregate result - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @param the value type of the resulting {@code KTable} * @see #aggregate(Initializer, Aggregator, Materialized) */ ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, @@ -135,14 +108,6 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator> materialized); /** - * Aggregate the values of records in this stream by the grouped key - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@code Aggregator} that computes a new aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @param the value type of the resulting {@code KTable} * @see #aggregate(Initializer, Aggregator, Named, Materialized) */ ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java index dd92292b..fe2b9788 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -45,11 +45,6 @@ public interface ImprovedKGroupedTable extends KGroupedTable { ImprovedKTable count(Materialized> materialized); /** - * Count number of records of the original {@code KTable} that got mapped to the same key into a new instance of - * {@code KTable} - * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store - * @return a {@code KTable} that contains "update" records with unmodified keys and Long values that represent the - * latest (rolling) count (i.e., number of records) for each key * @see #count(Materialized) */ ImprovedKTable count(ConfiguredMaterialized> materialized); @@ -58,12 +53,6 @@ public interface ImprovedKGroupedTable extends KGroupedTable { ImprovedKTable count(Named named, Materialized> materialized); /** - * Count number of records of the original {@code KTable} that got mapped to the same key into a new instance of - * {@code KTable} - * @param named the {@code Named} config used to name the processor in the topology - * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store - * @return a {@code KTable} that contains "update" records with unmodified keys and Long values that represent the - * latest (rolling) count (i.e., number of records) for each key * @see #count(Named, Materialized) */ ImprovedKTable count(Named named, @@ -80,13 +69,6 @@ ImprovedKTable reduce(Reducer adder, Reducer subtractor, Materialized> materialized); /** - * Combine the value of records of the original {@code KTable} that got mapped to the same key into a new - * instance of {@code KTable} - * @param adder a {@code Reducer} that adds a new value to the aggregate result - * @param subtractor a {@code Reducer} that removed an old value from the aggregate result - * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key * @see #reduce(Reducer, Reducer, Materialized) */ ImprovedKTable reduce(Reducer adder, Reducer subtractor, @@ -97,14 +79,6 @@ ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named Materialized> materialized); /** - * Combine the value of records of the original {@code KTable} that got mapped to the same key into a new - * instance of {@code KTable} - * @param adder a {@code Reducer} that adds a new value to the aggregate result - * @param subtractor a {@code Reducer} that removed an old value from the aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key * @see #reduce(Reducer, Reducer, Named, Materialized) */ ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, @@ -119,15 +93,6 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator> materialized); /** - * Aggregate the value of records of the original {@code KTable} that got mapped to the same key into a new - * instance of {@code KTable} - * @param initializer an {@code Initializer} that provides an initial aggregate result value - * @param adder an {@code Aggregator} that adds a new record to the aggregate result - * @param subtractor an {@code Aggregator} that removed an old record from the aggregate result - * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @param the value type of the aggregated {@code KTable} * @see #aggregate(Initializer, Aggregator, Aggregator, Materialized) */ ImprovedKTable aggregate(Initializer initializer, Aggregator adder, @@ -140,16 +105,6 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator> materialized); /** - * Aggregate the value of records of the original {@code KTable} that got mapped to the same key into a new - * instance of {@code KTable} - * @param initializer an {@code Initializer} that provides an initial aggregate result value - * @param adder an {@code Aggregator} that adds a new record to the aggregate result - * @param subtractor an {@code Aggregator} that removed an old record from the aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized the instance of {@code ConfiguredMaterialized} used to materialize the state store - * @return a {@code KTable} that contains "update" records with unmodified keys, and values that represent the - * latest (rolling) aggregate for each key - * @param the value type of the aggregated {@code KTable} * @see #aggregate(Initializer, Aggregator, Aggregator, Named, Materialized) */ ImprovedKTable aggregate(Initializer initializer, Aggregator adder, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 64b8ef25..bd229d5b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -242,29 +242,16 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKStream repartition(Repartitioned repartitioned); /** - * Materialize this stream to an auto-generated repartition topic and create a new KStream from the - * auto-generated topic - * @param repartitioned the {@code Repartitioned} instance used to specify {@code Serdes}, {@code StreamPartitioner} - * which determines how records are distributed among partitions of the topic, part of the topic name, and number - * of partitions - * for a repartition topic - * @return a {@code KStream} that contains the exact same repartitioned records as this {@code KStream} * @see #repartition(Repartitioned) */ ImprovedKStream repartition(ConfiguredRepartitioned repartitioned); /** - * Materialize this stream to a topic using the provided {@code ConfiguredProduced} instance - * @param topic the topic name - * @param produced the options to use when producing to the topic * @see #to(String, Produced) */ void to(String topic, ConfiguredProduced produced); /** - * Dynamically materialize this stream to topics using the provided {@code ConfiguredProduced} instance - * @param topicExtractor the extractor to determine the name of the Kafka topic to write to for each record - * @param produced the options to use when producing to the topic * @see #to(TopicNameExtractor, Produced) */ void to(TopicNameExtractor topicExtractor, ConfiguredProduced produced); @@ -297,11 +284,6 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKTable toTable(Materialized> materialized); /** - * Convert this stream to a {@code KTable} - * @param materialized an instance of {@code ConfiguredMaterialized} used to describe how the state store of the - * resulting table - * should be materialized - * @return a {@code KTable} that contains the same records as this {@code KStream} * @see #toTable(Materialized) */ ImprovedKTable toTable(ConfiguredMaterialized> materialized); @@ -310,12 +292,6 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKTable toTable(Named named, Materialized> materialized); /** - * Convert this stream to a {@code KTable} - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized an instance of {@code ConfiguredMaterialized} used to describe how the state store of the - * resulting table - * should be materialized - * @return a {@code KTable} that contains the same records as this {@code KStream} * @see #toTable(Named, Materialized) */ ImprovedKTable toTable(Named named, ConfiguredMaterialized> materialized); @@ -328,12 +304,6 @@ ImprovedKGroupedStream groupBy(KeyValueMapper grouped); /** - * Group the records of this {@code KStream} on a new key that is selected using the provided {@code KeyValueMapper} - * @param keySelector a {@code KeyValueMapper} that computes a new key for grouping - * @param grouped the {@code ConfiguredGrouped} instance used to specify {@code Serdes} and part of the name for a - * repartition topic if repartitioning is required - * @return a {@code KGroupedStream} that contains the grouped records of the original {@code KStream} - * @param the key type of the result {@code KGroupedStream} * @see #groupBy(KeyValueMapper, Grouped) */ ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, @@ -346,10 +316,6 @@ ImprovedKGroupedStream groupBy(KeyValueMapper groupByKey(Grouped grouped); /** - * Group the records by their current key into a {@code KGroupedStream} while preserving the original values - * @param grouped the {@code ConfiguredGrouped} instance used to specify {@code Serdes} and part of the name for a - * repartition topic if repartitioning is required - * @return a {@code KGroupedStream} that contains the grouped records of the original {@code KStream} * @see #groupByKey(Grouped) */ ImprovedKGroupedStream groupByKey(ConfiguredGrouped grouped); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java index 0f52e85c..22639cee 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -53,13 +53,6 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge Materialized> materialized); /** - * Aggregate the values of records in these streams by the grouped key and defined sessions - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param sessionMerger a {@code Merger} that combines two aggregation results - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window * @see #aggregate(Initializer, Merger, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, @@ -70,14 +63,6 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge Named named, Materialized> materialized); /** - * Aggregate the values of records in these streams by the grouped key and defined sessions - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param sessionMerger a {@code Merger} that combines two aggregation results - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window * @see #aggregate(Initializer, Merger, Named, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java index ca087433..fa876a6e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -54,11 +54,6 @@ public interface ImprovedSessionWindowedKStream extends SessionWindowedKSt ImprovedKTable, Long> count(Materialized> materialized); /** - * Count the number of records in this stream by the grouped key and defined sessions - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values - * that represent - * the latest (rolling) count (i.e., number of records) for each key per session * @see #count(Materialized) */ ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); @@ -68,12 +63,6 @@ ImprovedKTable, Long> count(Named named, Materialized> materialized); /** - * Count the number of records in this stream by the grouped key and defined sessions - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values - * that represent - * the latest (rolling) count (i.e., number of records) for each key per session * @see #count(Named, Materialized) */ ImprovedKTable, Long> count(Named named, @@ -95,14 +84,6 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Merger sessionMerger, Materialized> materialized); /** - * Aggregate the values of records in this stream by the grouped key and defined sessions - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@code Aggregator} that computes a new aggregate result - * @param sessionMerger a {@code Merger} that combines two aggregation results - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the latest (rolling) aggregate for each key per session - * @param the value type of the resulting {@code KTable} * @see #aggregate(Initializer, Aggregator, Merger, Materialized) */ ImprovedKTable, VR> aggregate(Initializer initializer, @@ -117,15 +98,6 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Materialized> materialized); /** - * Aggregate the values of records in this stream by the grouped key and defined sessions - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@code Aggregator} that computes a new aggregate result - * @param sessionMerger a {@code Merger} that combines two aggregation results - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the latest (rolling) aggregate for each key per session - * @param the value type of the resulting {@code KTable} * @see #aggregate(Initializer, Aggregator, Merger, Named, Materialized) */ ImprovedKTable, VR> aggregate(Initializer initializer, @@ -144,12 +116,6 @@ ImprovedKTable, V> reduce(Reducer reducer, Materialized> materialized); /** - * Combine the values of records in this stream by the grouped key and defined sessions - * @param reducer a {@code Reducer} that computes a new aggregate result - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key per session * @see #reduce(Reducer, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, @@ -160,13 +126,6 @@ ImprovedKTable, V> reduce(Reducer reducer, Named named, Materialized> materialized); /** - * Combine the values of records in this stream by the grouped key and defined sessions - * @param reducer a {@code Reducer} that computes a new aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key per session * @see #reduce(Reducer, Named, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, Named named, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java index 19f4affe..4bf4e000 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -51,12 +51,6 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Materialized> materialized); /** - * Aggregate the values of records in this stream by the grouped key and defined windows - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window * @see #aggregate(Initializer, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, @@ -67,13 +61,6 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Named Materialized> materialized); /** - * Aggregate the values of records in this stream by the grouped key and defined windows - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window * @see #aggregate(Initializer, Named, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java index 69d3ffd6..c2a4f665 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -53,12 +53,6 @@ public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream, Long> count(Materialized> materialized); /** - * Count the number of records in this stream by the grouped key and defined windows - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values - * that - * represent - * the latest (rolling) count (i.e., number of records) for each key within a window * @see #count(Materialized) */ ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); @@ -68,13 +62,6 @@ ImprovedKTable, Long> count(Named named, Materialized> materialized); /** - * Count the number of records in this stream by the grouped key and defined windows - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized an instance of {@code ConfiguredMaterialized} used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys and {@code Long} values - * that - * represent - * the latest (rolling) count (i.e., number of records) for each key within a window * @see #count(Named, Materialized) */ ImprovedKTable, Long> count(Named named, @@ -95,14 +82,6 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Materialized> materialized); /** - * Aggregate the values of records in this stream by the grouped key and defined windows - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@code Aggregator} that computes a new aggregate result - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window - * @param the value type of the resulting {@code KTable} * @see #aggregate(Initializer, Aggregator, Materialized) */ ImprovedKTable, VR> aggregate(Initializer initializer, @@ -115,15 +94,6 @@ ImprovedKTable, VR> aggregate(Initializer initializer, Named named, Materialized> materialized); /** - * Aggregate the values of records in this stream by the grouped key and defined windows - * @param initializer an {@code Initializer} that computes an initial intermediate aggregation result - * @param aggregator an {@code Aggregator} that computes a new aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window - * @param the value type of the resulting {@code KTable} * @see #aggregate(Initializer, Aggregator, Named, Materialized) */ ImprovedKTable, VR> aggregate(Initializer initializer, @@ -141,12 +111,6 @@ ImprovedKTable, V> reduce(Reducer reducer, Materialized> materialized); /** - * Combine the values of records in this stream by the grouped key and defined windows - * @param reducer a {@code Reducer} that computes a new aggregate result - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window * @see #reduce(Reducer, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, @@ -157,13 +121,6 @@ ImprovedKTable, V> reduce(Reducer reducer, Named named, Materialized> materialized); /** - * Combine the values of records in this stream by the grouped key and defined windows - * @param reducer a {@code Reducer} that computes a new aggregate result - * @param named a {@code Named} config used to name the processor in the topology - * @param materialized a {@code ConfiguredMaterialized} config used to materialize a state store - * @return a windowed {@code KTable} that contains "update" records with unmodified keys, and values that - * represent the - * latest (rolling) aggregate for each key within a window * @see #reduce(Reducer, Named, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, Named named, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index b89d5b05..d2ae9337 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -50,11 +50,6 @@ public class TopologyBuilder { Map kafkaProperties; /** - * Create a {@code KStream} from the specified topic - * @param topic the topic name - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(String) */ public ImprovedKStream stream(final String topic) { @@ -62,12 +57,6 @@ public ImprovedKStream stream(final String topic) { } /** - * Create a {@code KStream} from the specified topic - * @param topic the topic name - * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(String, Consumed) */ public ImprovedKStream stream(final String topic, final Consumed consumed) { @@ -75,12 +64,6 @@ public ImprovedKStream stream(final String topic, final Consumed type of keys - * @param type of values * @see StreamsBuilder#stream(String, Consumed) */ public ImprovedKStream stream(final String topic, final ConfiguredConsumed consumed) { @@ -88,11 +71,6 @@ public ImprovedKStream stream(final String topic, final ConfiguredC } /** - * Create a {@code KStream} from the specified topics - * @param topics the topic names - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(Collection) */ public ImprovedKStream stream(final Collection topics) { @@ -100,12 +78,6 @@ public ImprovedKStream stream(final Collection topics) { } /** - * Create a {@code KStream} from the specified topics - * @param topics the topic names - * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream stream(final Collection topics, final Consumed consumed) { @@ -113,12 +85,6 @@ public ImprovedKStream stream(final Collection topics, fina } /** - * Create a {@code KStream} from the specified topics - * @param topics the topic names - * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream stream(final Collection topics, @@ -127,11 +93,6 @@ public ImprovedKStream stream(final Collection topics, } /** - * Create a {@code KStream} from the specified topic pattern - * @param topicPattern the pattern to match for topic names - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(Pattern) */ public ImprovedKStream stream(final Pattern topicPattern) { @@ -139,12 +100,6 @@ public ImprovedKStream stream(final Pattern topicPattern) { } /** - * Create a {@code KStream} from the specified topic pattern - * @param topicPattern the pattern to match for topic names - * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream stream(final Pattern topicPattern, final Consumed consumed) { @@ -152,12 +107,6 @@ public ImprovedKStream stream(final Pattern topicPattern, final Con } /** - * Create a {@code KStream} from the specified topic pattern - * @param topicPattern the pattern to match for topic names - * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} - * @param type of keys - * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream stream(final Pattern topicPattern, final ConfiguredConsumed consumed) { From a3e2417670e1478e28f8030b3bf153e16d7e98ad Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 15:19:19 +0100 Subject: [PATCH 24/72] Add tests --- .../bakdata/kafka/ImprovedKStreamTest.java | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index b9318103..7497465f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -27,6 +27,7 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.bakdata.fluent_kafka_streams_tests.TestTopology; import io.vavr.collection.List; @@ -36,6 +37,7 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.processor.api.FixedKeyProcessor; @@ -221,6 +223,248 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldMap() { + final KeyValueMapper> mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn(new KeyValue<>("baz", "qux")); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.map(mapper).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValues() { + final ValueMapper mapper = mock(); + when(mapper.apply("bar")).thenReturn("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.mapValues(mapper).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValuesWithKey() { + final ValueMapperWithKey mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.mapValues(mapper).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFlatMap() { + final KeyValueMapper>> mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn(List.of(new KeyValue<>("baz", "qux"))); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.flatMap(mapper).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFlatMapValues() { + final ValueMapper> mapper = mock(); + when(mapper.apply("bar")).thenReturn(List.of("baz")); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.flatMapValues(mapper).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFlatMapValuesWithKey() { + final ValueMapperWithKey> mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn(List.of("baz")); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.flatMapValues(mapper).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldProcess() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + this.forward(inputRecord.withKey("baz").withValue("qux")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream processed = input.process(processor); + processed.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldProcessValues() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + this.forward(inputRecord.withValue("baz")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream processed = input.processValues(processor); + processed.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFilter() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("foo", "baz")).thenReturn(false); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.filter(predicate).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldSelectKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.selectKey((k, v) -> v).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldMapCapturingErrors() { final KeyValueMapper> mapper = mock(); From 0a5528580b9c72e6e79443f1d54c38239e302309 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 15:58:15 +0100 Subject: [PATCH 25/72] Add tests --- .../bakdata/kafka/ImprovedKStreamTest.java | 247 +++++++++++++++++- 1 file changed, 242 insertions(+), 5 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index 7497465f..a985359f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -444,6 +444,31 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldFilterNot() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("foo", "baz")).thenReturn(true); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + input.filterNot(predicate).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldSelectKey() { final StreamsApp app = new SimpleApp() { @@ -846,6 +871,28 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldConvertToTable() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKTable table = input.toTable(); + table.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldConvertToTableUsingMaterialized() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -877,6 +924,196 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldJoin() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream otherInput = builder.stream("other_input"); + final ImprovedKStream joined = input.join(otherInput, + (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldJoinWithKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream otherInput = builder.stream("other_input"); + final ImprovedKStream joined = input.join(otherInput, + (k, v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldLeftJoin() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream otherInput = builder.stream("other_input"); + final ImprovedKStream joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("other_input") + .at(0L) + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldLeftJoinWithKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream otherInput = builder.stream("other_input"); + final ImprovedKStream joined = input.leftJoin(otherInput, + (k, v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("other_input") + .at(0L) + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldOuterJoin() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream otherInput = builder.stream("other_input"); + final ImprovedKStream joined = input.outerJoin(otherInput, + (v1, v2) -> v1 == null ? v2 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("input") + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldOuterJoinWithKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final ImprovedKStream input = builder.stream("input"); + final ImprovedKStream otherInput = builder.stream("other_input"); + final ImprovedKStream joined = input.outerJoin(otherInput, + (k, v1, v2) -> v1 == null ? v2 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("input") + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldJoinUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -916,7 +1153,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldJoinWithKey() { + void shouldJoinWithKeyUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -956,7 +1193,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldLeftJoin() { + void shouldLeftJoinUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -1005,7 +1242,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldLeftJoinWithKey() { + void shouldLeftJoinWithKeyUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -1054,7 +1291,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldOuterJoin() { + void shouldOuterJoinUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -1103,7 +1340,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldOuterJoinWithKey() { + void shouldOuterJoinWithKeyUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { From 8d7c664d15fe2b7a7fb40fdf73a05a821868969f Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 16:08:12 +0100 Subject: [PATCH 26/72] Add docs --- .../com/bakdata/kafka/ConfiguredConsumed.java | 13 +----- .../com/bakdata/kafka/ConfiguredGrouped.java | 25 ++--------- .../com/bakdata/kafka/ConfiguredJoined.java | 37 +++------------- .../bakdata/kafka/ConfiguredMaterialized.java | 22 ++-------- .../com/bakdata/kafka/ConfiguredProduced.java | 25 ++--------- .../kafka/ConfiguredRepartitioned.java | 29 +++---------- .../com/bakdata/kafka/ConfiguredStores.java | 42 ------------------- .../bakdata/kafka/ConfiguredStreamJoined.java | 26 ++---------- 8 files changed, 27 insertions(+), 192 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java index e06f7c2f..af72e2a6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java @@ -72,12 +72,7 @@ public static ConfiguredConsumed valueSerde(final Preconfigured type of keys - * @param type of values + * @see Consumed#with(Serde, Serde) */ public static ConfiguredConsumed with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -85,11 +80,7 @@ public static ConfiguredConsumed with(final Preconfigured> } /** - * Create an instance of {@code ConfiguredConsumed} with provided processor name - * @param processorName the processor name to be used - * @return a new instance of {@code ConfiguredConsumed} - * @param type of keys - * @param type of values + * @see Consumed#as(String) */ public static ConfiguredConsumed as(final String processorName) { return new ConfiguredConsumed<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java index 8072cd08..731600ea 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java @@ -48,34 +48,21 @@ public final class ConfiguredGrouped { private final String name; /** - * Create an instance of {@code ConfiguredGrouped} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredGrouped} - * @param type of keys - * @param type of values + * @see Grouped#keySerde(Serde) */ public static ConfiguredGrouped keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredGrouped} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredGrouped} - * @param type of keys - * @param type of values + * @see Grouped#valueSerde(Serde) */ public static ConfiguredGrouped valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } /** - * Create an instance of {@code ConfiguredGrouped} with provided key and value serde - * @param keySerde Serde to use for keys - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredGrouped} - * @param type of keys - * @param type of values + * @see Grouped#with(Serde, Serde) */ public static ConfiguredGrouped with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -83,11 +70,7 @@ public static ConfiguredGrouped with(final Preconfigured> } /** - * Create an instance of {@code ConfiguredGrouped} with provided name - * @param name the name used for a repartition topic if required - * @return a new instance of {@code ConfiguredGrouped} - * @param type of keys - * @param type of values + * @see Grouped#as(String) */ public static ConfiguredGrouped as(final String name) { return new ConfiguredGrouped<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), name); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java index 134bb568..05826971 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java @@ -50,12 +50,7 @@ public final class ConfiguredJoined { private final Duration gracePeriod; /** - * Create an instance of {@code ConfiguredJoined} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see Joined#keySerde(Serde) */ public static ConfiguredJoined keySerde( final Preconfigured> keySerde) { @@ -63,12 +58,7 @@ public static ConfiguredJoined keySerde( } /** - * Create an instance of {@code ConfiguredJoined} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see Joined#valueSerde(Serde) */ public static ConfiguredJoined valueSerde( final Preconfigured> valueSerde) { @@ -76,12 +66,7 @@ public static ConfiguredJoined valueSerde( } /** - * Create an instance of {@code ConfiguredJoined} with provided other value serde - * @param valueSerde Serde to use for other values - * @return a new instance of {@code ConfiguredJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see Joined#otherValueSerde(Serde) */ public static ConfiguredJoined otherValueSerde( final Preconfigured> valueSerde) { @@ -89,14 +74,7 @@ public static ConfiguredJoined otherValueSerde( } /** - * Create an instance of {@code ConfiguredJoined} with provided key and value serde - * @param keySerde Serde to use for keys - * @param valueSerde Serde to use for values - * @param otherValueSerde Serde to use for other values - * @return a new instance of {@code ConfiguredJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see Joined#with(Serde, Serde, Serde) */ public static ConfiguredJoined with( final Preconfigured> keySerde, @@ -106,12 +84,7 @@ public static ConfiguredJoined with( } /** - * Create an instance of {@code ConfiguredJoined} with provided store name - * @param name the name used as the base for naming components of the join including any repartition topics - * @return a new instance of {@code ConfiguredJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see Joined#as(String) */ public static ConfiguredJoined as(final String name) { return new ConfiguredJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java index 7214768e..4d6e7a7c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java @@ -86,13 +86,7 @@ public static ConfiguredMaterialized value } /** - * Create an instance of {@code ConfiguredMaterialized} with provided key and value serde - * @param keySerde Serde to use for keys - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredMaterialized} - * @param type of keys - * @param type of values - * @param type of state store + * @see Materialized#with(Serde, Serde) */ public static ConfiguredMaterialized with( final Preconfigured> keySerde, @@ -101,12 +95,7 @@ public static ConfiguredMaterialized with( } /** - * Create an instance of {@code ConfiguredMaterialized} with provided store name - * @param storeName the store name to be used - * @return a new instance of {@code ConfiguredMaterialized} - * @param type of keys - * @param type of values - * @param type of state store + * @see Materialized#as(String) */ public static ConfiguredMaterialized as(final String storeName) { return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeName, null, @@ -114,12 +103,7 @@ public static ConfiguredMaterialized as(fi } /** - * Create an instance of {@code ConfiguredMaterialized} with provided store suppliers - * @param storeSuppliers the store suppliers to be used - * @return a new instance of {@code ConfiguredMaterialized} - * @param type of keys - * @param type of values - * @param type of state store + * @see Materialized#as(DslStoreSuppliers) */ public static ConfiguredMaterialized as( final DslStoreSuppliers storeSuppliers) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java index 6d365b42..b120daef 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java @@ -48,34 +48,21 @@ public final class ConfiguredProduced { private final String name; /** - * Create an instance of {@code ConfiguredProduced} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredProduced} - * @param type of keys - * @param type of values + * @see Produced#keySerde(Serde) */ public static ConfiguredProduced keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredProduced} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredProduced} - * @param type of keys - * @param type of values + * @see Produced#valueSerde(Serde) */ public static ConfiguredProduced valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } /** - * Create an instance of {@code ConfiguredProduced} with provided key and value serde - * @param keySerde Serde to use for keys - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredProduced} - * @param type of keys - * @param type of values + * @see Produced#with(Serde, Serde) */ public static ConfiguredProduced with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -83,11 +70,7 @@ public static ConfiguredProduced with(final Preconfigured> } /** - * Create an instance of {@code ConfiguredProduced} with provided processor name - * @param processorName the processor name to be used - * @return a new instance of {@code ConfiguredProduced} - * @param type of keys - * @param type of values + * @see Produced#as(String) */ public static ConfiguredProduced as(final String processorName) { return new ConfiguredProduced<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java index 0c8a60d7..98318c82 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java @@ -74,12 +74,7 @@ public static ConfiguredRepartitioned valueSerde(final Preconfigure } /** - * Create an instance of {@code ConfiguredRepartitioned} with provided key and value serde - * @param keySerde Serde to use for keys - * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredRepartitioned} - * @param type of keys - * @param type of values + * @see Repartitioned#with(Serde, Serde) */ public static ConfiguredRepartitioned with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { @@ -87,11 +82,7 @@ public static ConfiguredRepartitioned with(final Preconfigured type of keys - * @param type of values + * @see Repartitioned#as(String) */ public static ConfiguredRepartitioned as(final String name) { return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, name, @@ -99,11 +90,7 @@ public static ConfiguredRepartitioned as(final String name) { } /** - * Create an instance of {@code ConfiguredRepartitioned} with provided number of partitions for repartition topic - * @param numberOfPartitions number of partitions - * @return a new instance of {@code ConfiguredRepartitioned} - * @param type of keys - * @param type of values + * @see Repartitioned#numberOfPartitions(int) */ public static ConfiguredRepartitioned numberOfPartitions(final int numberOfPartitions) { return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, @@ -111,11 +98,7 @@ public static ConfiguredRepartitioned numberOfPartitions(final int } /** - * Create an instance of {@code ConfiguredRepartitioned} with provided partitioner for repartition topic - * @param partitioner partitioner to use - * @return a new instance of {@code ConfiguredRepartitioned} - * @param type of keys - * @param type of values + * @see Repartitioned#streamPartitioner(StreamPartitioner) */ public static ConfiguredRepartitioned streamPartitioner(final StreamPartitioner partitioner) { return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), partitioner, @@ -123,9 +106,7 @@ public static ConfiguredRepartitioned streamPartitioner(final Strea } /** - * Create an instance of {@code ConfiguredRepartitioned} with provided number of partitions for repartition topic - * @param numberOfPartitions number of partitions - * @return a new instance of {@code ConfiguredRepartitioned} + * @see Repartitioned#withNumberOfPartitions(int) */ public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPartitions) { return new ConfiguredRepartitioned<>(this.keySerde, this.valueSerde, this.streamPartitioner, this.name, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java index 3593a58f..b5a43215 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java @@ -49,13 +49,6 @@ public class ConfiguredStores { private final @NonNull Configurator configurator; /** - * Creates a {@code StoreBuilder} that can be used to build a {@code SessionStore} - * @param supplier a {@code SessionBytesStoreSupplier} - * @param keySerde the key serde to use - * @param valueSerde the value serde to use - * @return a store builder - * @param key type - * @param value type * @see Stores#sessionStoreBuilder(SessionBytesStoreSupplier, Serde, Serde) */ public StoreBuilder> sessionStoreBuilder(final SessionBytesStoreSupplier supplier, @@ -65,13 +58,6 @@ public StoreBuilder> sessionStoreBuilder(final Session } /** - * Creates a {@code StoreBuilder} that can be used to build a {@code TimestampedWindowStore} - * @param supplier a {@code WindowBytesStoreSupplier} - * @param keySerde the key serde to use - * @param valueSerde the value serde to use - * @return a store builder - * @param key type - * @param value type * @see Stores#timestampedWindowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) */ public StoreBuilder> timestampedWindowStoreBuilder( @@ -82,13 +68,6 @@ public StoreBuilder> timestampedWindowStoreB } /** - * Creates a {@code StoreBuilder} that can be used to build a {@code WindowStore} - * @param supplier a {@code WindowBytesStoreSupplier} - * @param keySerde the key serde to use - * @param valueSerde the value serde to use - * @return a store builder - * @param key type - * @param value type * @see Stores#windowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) */ public StoreBuilder> windowStoreBuilder(final WindowBytesStoreSupplier supplier, @@ -98,13 +77,6 @@ public StoreBuilder> windowStoreBuilder(final WindowByt } /** - * Creates a {@code StoreBuilder} that can be used to build a {@code VersionedKeyValueStore} - * @param supplier a {@code VersionedBytesStoreSupplier} - * @param keySerde the key serde to use - * @param valueSerde the value serde to use - * @return a store builder - * @param key type - * @param value type * @see Stores#versionedKeyValueStoreBuilder(VersionedBytesStoreSupplier, Serde, Serde) */ public StoreBuilder> versionedKeyValueStoreBuilder( @@ -115,13 +87,6 @@ public StoreBuilder> versionedKeyValueStoreB } /** - * Creates a {@code StoreBuilder} that can be used to build a {@code TimestampedKeyValueStore} - * @param supplier a {@code KeyValueBytesStoreSupplier} - * @param keySerde the key serde to use - * @param valueSerde the value serde to use - * @return a store builder - * @param key type - * @param value type * @see Stores#timestampedKeyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) */ public StoreBuilder> timestampedKeyValueStoreBuilder( @@ -132,13 +97,6 @@ public StoreBuilder> timestampedKeyValueSt } /** - * Creates a {@code StoreBuilder} that can be used to build a {@code KeyValueStore} - * @param supplier a {@code KeyValueBytesStoreSupplier} - * @param keySerde the key serde to use - * @param valueSerde the value serde to use - * @return a store builder - * @param key type - * @param value type * @see Stores#keyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) */ public StoreBuilder> keyValueStoreBuilder(final KeyValueBytesStoreSupplier supplier, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java index bc977c55..377dd72e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java @@ -99,14 +99,7 @@ public static ConfiguredStreamJoined otherValueSerde( } /** - * Create an instance of {@code ConfiguredStreamJoined} with provided key and value serde - * @param keySerde Serde to use for keys - * @param valueSerde Serde to use for values - * @param otherValueSerde Serde to use for other values - * @return a new instance of {@code ConfiguredStreamJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see StreamJoined#with(Serde, Serde, Serde) */ public static ConfiguredStreamJoined with( final Preconfigured> keySerde, @@ -117,12 +110,7 @@ public static ConfiguredStreamJoined with( } /** - * Create an instance of {@code ConfiguredStreamJoined} with provided store name - * @param storeName the store name to be used - * @return a new instance of {@code ConfiguredStreamJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see StreamJoined#as(String) */ public static ConfiguredStreamJoined as(final String storeName) { return new ConfiguredStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), @@ -131,15 +119,9 @@ public static ConfiguredStreamJoined as(final String stor } /** - * Create an instance of {@code ConfiguredStreamJoined} with provided store suppliers - * @param storeSuppliers the store suppliers to be used - * @return a new instance of {@code ConfiguredStreamJoined} - * @param type of keys - * @param this value type - * @param other value type + * @see StreamJoined#with(DslStoreSuppliers) */ - public static ConfiguredStreamJoined as( - final DslStoreSuppliers storeSuppliers) { + public static ConfiguredStreamJoined with(final DslStoreSuppliers storeSuppliers) { return new ConfiguredStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeSuppliers, null, null, new HashMap<>(), true); From e2629d97d9cba724181616bbe9d40768b60dd90e Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 16:19:42 +0100 Subject: [PATCH 27/72] Rename --- ...figuredConsumed.java => AutoConsumed.java} | 22 ++--- ...onfiguredGrouped.java => AutoGrouped.java} | 14 ++-- ...{ConfiguredJoined.java => AutoJoined.java} | 16 ++-- ...aterialized.java => AutoMaterialized.java} | 26 +++--- ...figuredProduced.java => AutoProduced.java} | 14 ++-- ...artitioned.java => AutoRepartitioned.java} | 34 ++++---- ...{ConfiguredStores.java => AutoStores.java} | 2 +- ...treamJoined.java => AutoStreamJoined.java} | 32 +++---- .../kafka/ImprovedCogroupedKStream.java | 4 +- .../kafka/ImprovedCogroupedStreamImpl.java | 4 +- .../bakdata/kafka/ImprovedKGroupedStream.java | 12 +-- .../kafka/ImprovedKGroupedStreamImpl.java | 12 +-- .../bakdata/kafka/ImprovedKGroupedTable.java | 12 +-- .../kafka/ImprovedKGroupedTableImpl.java | 12 +-- .../com/bakdata/kafka/ImprovedKStream.java | 40 ++++----- .../bakdata/kafka/ImprovedKStreamImpl.java | 40 ++++----- .../com/bakdata/kafka/ImprovedKTable.java | 46 +++++----- .../com/bakdata/kafka/ImprovedKTableImpl.java | 46 +++++----- ...provedSessionWindowedCogroupedKStream.java | 4 +- ...vedSessionWindowedCogroupedStreamImpl.java | 4 +- .../kafka/ImprovedSessionWindowedKStream.java | 12 +-- .../ImprovedSessionWindowedStreamImpl.java | 12 +-- .../ImprovedTimeWindowedCogroupedKStream.java | 4 +- ...provedTimeWindowedCogroupedStreamImpl.java | 4 +- .../kafka/ImprovedTimeWindowedKStream.java | 12 +-- .../kafka/ImprovedTimeWindowedStreamImpl.java | 12 +-- .../com/bakdata/kafka/TopologyBuilder.java | 20 ++--- .../kafka/ImprovedKGroupedStreamTest.java | 16 ++-- .../bakdata/kafka/ImprovedKStreamTest.java | 84 +++++++++---------- .../bakdata/kafka/TopologyBuilderTest.java | 16 ++-- .../MirrorWithNonDefaultSerde.java | 8 +- 31 files changed, 298 insertions(+), 298 deletions(-) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredConsumed.java => AutoConsumed.java} (76%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredGrouped.java => AutoGrouped.java} (80%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredJoined.java => AutoJoined.java} (85%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredMaterialized.java => AutoMaterialized.java} (78%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredProduced.java => AutoProduced.java} (81%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredRepartitioned.java => AutoRepartitioned.java} (68%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredStores.java => AutoStores.java} (99%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ConfiguredStreamJoined.java => AutoStreamJoined.java} (77%) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java similarity index 76% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java index af72e2a6..e7dc4b62 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredConsumed.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java @@ -41,7 +41,7 @@ */ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredConsumed { +public final class AutoConsumed { private final @NonNull Preconfigured> keySerde; private final @NonNull Preconfigured> valueSerde; @@ -50,40 +50,40 @@ public final class ConfiguredConsumed { private final String name; /** - * Create an instance of {@code ConfiguredConsumed} with provided key serde + * Create an instance of {@code AutoConsumed} with provided key serde * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredConsumed} + * @return a new instance of {@code AutoConsumed} * @param type of keys * @param type of values */ - public static ConfiguredConsumed keySerde(final Preconfigured> keySerde) { + public static AutoConsumed keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredConsumed} with provided value serde + * Create an instance of {@code AutoConsumed} with provided value serde * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredConsumed} + * @return a new instance of {@code AutoConsumed} * @param type of keys * @param type of values */ - public static ConfiguredConsumed valueSerde(final Preconfigured> valueSerde) { + public static AutoConsumed valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } /** * @see Consumed#with(Serde, Serde) */ - public static ConfiguredConsumed with(final Preconfigured> keySerde, + public static AutoConsumed with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { - return new ConfiguredConsumed<>(keySerde, valueSerde, null, null, null); + return new AutoConsumed<>(keySerde, valueSerde, null, null, null); } /** * @see Consumed#as(String) */ - public static ConfiguredConsumed as(final String processorName) { - return new ConfiguredConsumed<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, + public static AutoConsumed as(final String processorName) { + return new AutoConsumed<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, processorName); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java similarity index 80% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java index 731600ea..6c383278 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredGrouped.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java @@ -38,7 +38,7 @@ * @see Grouped */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredGrouped { +public final class AutoGrouped { @With private final @NonNull Preconfigured> keySerde; @@ -50,30 +50,30 @@ public final class ConfiguredGrouped { /** * @see Grouped#keySerde(Serde) */ - public static ConfiguredGrouped keySerde(final Preconfigured> keySerde) { + public static AutoGrouped keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** * @see Grouped#valueSerde(Serde) */ - public static ConfiguredGrouped valueSerde(final Preconfigured> valueSerde) { + public static AutoGrouped valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } /** * @see Grouped#with(Serde, Serde) */ - public static ConfiguredGrouped with(final Preconfigured> keySerde, + public static AutoGrouped with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { - return new ConfiguredGrouped<>(keySerde, valueSerde, null); + return new AutoGrouped<>(keySerde, valueSerde, null); } /** * @see Grouped#as(String) */ - public static ConfiguredGrouped as(final String name) { - return new ConfiguredGrouped<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), name); + public static AutoGrouped as(final String name) { + return new AutoGrouped<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), name); } Grouped configure(final Configurator configurator) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java similarity index 85% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java index 05826971..a7087799 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredJoined.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java @@ -41,7 +41,7 @@ */ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredJoined { +public final class AutoJoined { private final @NonNull Preconfigured> keySerde; private final @NonNull Preconfigured> valueSerde; @@ -52,7 +52,7 @@ public final class ConfiguredJoined { /** * @see Joined#keySerde(Serde) */ - public static ConfiguredJoined keySerde( + public static AutoJoined keySerde( final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); } @@ -60,7 +60,7 @@ public static ConfiguredJoined keySerde( /** * @see Joined#valueSerde(Serde) */ - public static ConfiguredJoined valueSerde( + public static AutoJoined valueSerde( final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); } @@ -68,7 +68,7 @@ public static ConfiguredJoined valueSerde( /** * @see Joined#otherValueSerde(Serde) */ - public static ConfiguredJoined otherValueSerde( + public static AutoJoined otherValueSerde( final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), valueSerde); } @@ -76,18 +76,18 @@ public static ConfiguredJoined otherValueSerde( /** * @see Joined#with(Serde, Serde, Serde) */ - public static ConfiguredJoined with( + public static AutoJoined with( final Preconfigured> keySerde, final Preconfigured> valueSerde, final Preconfigured> otherValueSerde) { - return new ConfiguredJoined<>(keySerde, valueSerde, otherValueSerde, null, null); + return new AutoJoined<>(keySerde, valueSerde, otherValueSerde, null, null); } /** * @see Joined#as(String) */ - public static ConfiguredJoined as(final String name) { - return new ConfiguredJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), + public static AutoJoined as(final String name) { + return new AutoJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), name, null); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java similarity index 78% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java index 4d6e7a7c..d434554c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredMaterialized.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java @@ -44,7 +44,7 @@ * @see Materialized */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredMaterialized { +public final class AutoMaterialized { @With private final @NonNull Preconfigured> keySerde; @@ -60,27 +60,27 @@ public final class ConfiguredMaterialized { private final boolean cachingEnabled; /** - * Create an instance of {@code ConfiguredMaterialized} with provided key serde + * Create an instance of {@code AutoMaterialized} with provided key serde * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredMaterialized} + * @return a new instance of {@code AutoMaterialized} * @param type of keys * @param type of values * @param type of state store */ - public static ConfiguredMaterialized keySerde( + public static AutoMaterialized keySerde( final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredMaterialized} with provided value serde + * Create an instance of {@code AutoMaterialized} with provided value serde * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredMaterialized} + * @return a new instance of {@code AutoMaterialized} * @param type of keys * @param type of values * @param type of state store */ - public static ConfiguredMaterialized valueSerde( + public static AutoMaterialized valueSerde( final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } @@ -88,26 +88,26 @@ public static ConfiguredMaterialized value /** * @see Materialized#with(Serde, Serde) */ - public static ConfiguredMaterialized with( + public static AutoMaterialized with( final Preconfigured> keySerde, final Preconfigured> valueSerde) { - return new ConfiguredMaterialized<>(keySerde, valueSerde, null, null, null, new HashMap<>(), true, true); + return new AutoMaterialized<>(keySerde, valueSerde, null, null, null, new HashMap<>(), true, true); } /** * @see Materialized#as(String) */ - public static ConfiguredMaterialized as(final String storeName) { - return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeName, null, + public static AutoMaterialized as(final String storeName) { + return new AutoMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeName, null, null, new HashMap<>(), true, true); } /** * @see Materialized#as(DslStoreSuppliers) */ - public static ConfiguredMaterialized as( + public static AutoMaterialized as( final DslStoreSuppliers storeSuppliers) { - return new ConfiguredMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, + return new AutoMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, storeSuppliers, new HashMap<>(), true, true); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java similarity index 81% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java index b120daef..d98bb710 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredProduced.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java @@ -40,7 +40,7 @@ */ @With @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredProduced { +public final class AutoProduced { private final @NonNull Preconfigured> keySerde; private final @NonNull Preconfigured> valueSerde; @@ -50,30 +50,30 @@ public final class ConfiguredProduced { /** * @see Produced#keySerde(Serde) */ - public static ConfiguredProduced keySerde(final Preconfigured> keySerde) { + public static AutoProduced keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** * @see Produced#valueSerde(Serde) */ - public static ConfiguredProduced valueSerde(final Preconfigured> valueSerde) { + public static AutoProduced valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } /** * @see Produced#with(Serde, Serde) */ - public static ConfiguredProduced with(final Preconfigured> keySerde, + public static AutoProduced with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { - return new ConfiguredProduced<>(keySerde, valueSerde, null, null); + return new AutoProduced<>(keySerde, valueSerde, null, null); } /** * @see Produced#as(String) */ - public static ConfiguredProduced as(final String processorName) { - return new ConfiguredProduced<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, + public static AutoProduced as(final String processorName) { + return new AutoProduced<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, processorName); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java similarity index 68% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java index 98318c82..a63db629 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredRepartitioned.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java @@ -39,7 +39,7 @@ * @see Repartitioned */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredRepartitioned { +public final class AutoRepartitioned { @With private final @NonNull Preconfigured> keySerde; @@ -52,64 +52,64 @@ public final class ConfiguredRepartitioned { private final Integer numberOfPartitions; /** - * Create an instance of {@code ConfiguredRepartitioned} with provided key serde + * Create an instance of {@code AutoRepartitioned} with provided key serde * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredRepartitioned} + * @return a new instance of {@code AutoRepartitioned} * @param type of keys * @param type of values */ - public static ConfiguredRepartitioned keySerde(final Preconfigured> keySerde) { + public static AutoRepartitioned keySerde(final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredRepartitioned} with provided value serde + * Create an instance of {@code AutoRepartitioned} with provided value serde * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredRepartitioned} + * @return a new instance of {@code AutoRepartitioned} * @param type of keys * @param type of values */ - public static ConfiguredRepartitioned valueSerde(final Preconfigured> valueSerde) { + public static AutoRepartitioned valueSerde(final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde); } /** * @see Repartitioned#with(Serde, Serde) */ - public static ConfiguredRepartitioned with(final Preconfigured> keySerde, + public static AutoRepartitioned with(final Preconfigured> keySerde, final Preconfigured> valueSerde) { - return new ConfiguredRepartitioned<>(keySerde, valueSerde, null, null, null); + return new AutoRepartitioned<>(keySerde, valueSerde, null, null, null); } /** * @see Repartitioned#as(String) */ - public static ConfiguredRepartitioned as(final String name) { - return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, name, + public static AutoRepartitioned as(final String name) { + return new AutoRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, name, null); } /** * @see Repartitioned#numberOfPartitions(int) */ - public static ConfiguredRepartitioned numberOfPartitions(final int numberOfPartitions) { - return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, + public static AutoRepartitioned numberOfPartitions(final int numberOfPartitions) { + return new AutoRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, numberOfPartitions); } /** * @see Repartitioned#streamPartitioner(StreamPartitioner) */ - public static ConfiguredRepartitioned streamPartitioner(final StreamPartitioner partitioner) { - return new ConfiguredRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), partitioner, + public static AutoRepartitioned streamPartitioner(final StreamPartitioner partitioner) { + return new AutoRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), partitioner, null, null); } /** * @see Repartitioned#withNumberOfPartitions(int) */ - public ConfiguredRepartitioned withNumberOfPartitions(final int numberOfPartitions) { - return new ConfiguredRepartitioned<>(this.keySerde, this.valueSerde, this.streamPartitioner, this.name, + public AutoRepartitioned withNumberOfPartitions(final int numberOfPartitions) { + return new AutoRepartitioned<>(this.keySerde, this.valueSerde, this.streamPartitioner, this.name, numberOfPartitions); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStores.java similarity index 99% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStores.java index b5a43215..02a94d5e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStores.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStores.java @@ -44,7 +44,7 @@ * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Stores} using {@link Configurator} */ @RequiredArgsConstructor -public class ConfiguredStores { +public class AutoStores { private final @NonNull Configurator configurator; diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java similarity index 77% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java index 377dd72e..84de3992 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConfiguredStreamJoined.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java @@ -42,7 +42,7 @@ * @see StreamJoined */ @RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class ConfiguredStreamJoined { +public final class AutoStreamJoined { @With private final @NonNull Preconfigured> keySerde; @@ -60,40 +60,40 @@ public final class ConfiguredStreamJoined { private final boolean loggingEnabled; /** - * Create an instance of {@code ConfiguredStreamJoined} with provided key serde + * Create an instance of {@code AutoStreamJoined} with provided key serde * @param keySerde Serde to use for keys - * @return a new instance of {@code ConfiguredStreamJoined} + * @return a new instance of {@code AutoStreamJoined} * @param type of keys * @param this value type * @param other value type */ - public static ConfiguredStreamJoined keySerde( + public static AutoStreamJoined keySerde( final Preconfigured> keySerde) { return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredStreamJoined} with provided value serde + * Create an instance of {@code AutoStreamJoined} with provided value serde * @param valueSerde Serde to use for values - * @return a new instance of {@code ConfiguredStreamJoined} + * @return a new instance of {@code AutoStreamJoined} * @param type of keys * @param this value type * @param other value type */ - public static ConfiguredStreamJoined valueSerde( + public static AutoStreamJoined valueSerde( final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); } /** - * Create an instance of {@code ConfiguredStreamJoined} with provided other value serde + * Create an instance of {@code AutoStreamJoined} with provided other value serde * @param valueSerde Serde to use for other values - * @return a new instance of {@code ConfiguredStreamJoined} + * @return a new instance of {@code AutoStreamJoined} * @param type of keys * @param this value type * @param other value type */ - public static ConfiguredStreamJoined otherValueSerde( + public static AutoStreamJoined otherValueSerde( final Preconfigured> valueSerde) { return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), valueSerde); } @@ -101,19 +101,19 @@ public static ConfiguredStreamJoined otherValueSerde( /** * @see StreamJoined#with(Serde, Serde, Serde) */ - public static ConfiguredStreamJoined with( + public static AutoStreamJoined with( final Preconfigured> keySerde, final Preconfigured> valueSerde, final Preconfigured> otherValueSerde) { - return new ConfiguredStreamJoined<>(keySerde, valueSerde, otherValueSerde, null, null, null, new HashMap<>(), + return new AutoStreamJoined<>(keySerde, valueSerde, otherValueSerde, null, null, null, new HashMap<>(), true); } /** * @see StreamJoined#as(String) */ - public static ConfiguredStreamJoined as(final String storeName) { - return new ConfiguredStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), + public static AutoStreamJoined as(final String storeName) { + return new AutoStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, storeName, new HashMap<>(), true); } @@ -121,8 +121,8 @@ public static ConfiguredStreamJoined as(final String stor /** * @see StreamJoined#with(DslStoreSuppliers) */ - public static ConfiguredStreamJoined with(final DslStoreSuppliers storeSuppliers) { - return new ConfiguredStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), + public static AutoStreamJoined with(final DslStoreSuppliers storeSuppliers) { + return new AutoStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeSuppliers, null, null, new HashMap<>(), true); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java index 504b4f84..bab48523 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java @@ -63,7 +63,7 @@ ImprovedKTable aggregate(Initializer initializer, * @see #aggregate(Initializer, Materialized) */ ImprovedKTable aggregate(Initializer initializer, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable aggregate(Initializer initializer, Named named, @@ -73,7 +73,7 @@ ImprovedKTable aggregate(Initializer initializer, Named named, * @see #aggregate(Initializer, Named, Materialized) */ ImprovedKTable aggregate(Initializer initializer, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedTimeWindowedCogroupedKStream windowedBy(Windows windows); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java index b2b8cc49..5e709e3d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java @@ -70,7 +70,7 @@ public ImprovedKTable aggregate(final Initializer initializer, @Override public ImprovedKTable aggregate(final Initializer initializer, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); } @@ -82,7 +82,7 @@ public ImprovedKTable aggregate(final Initializer initializer, final Na @Override public ImprovedKTable aggregate(final Initializer initializer, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index bbbba7ae..88aa129c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -57,7 +57,7 @@ public interface ImprovedKGroupedStream extends KGroupedStream { /** * @see #count(Materialized) */ - ImprovedKTable count(ConfiguredMaterialized> materialized); + ImprovedKTable count(AutoMaterialized> materialized); @Override ImprovedKTable count(Named named, Materialized> materialized); @@ -66,7 +66,7 @@ public interface ImprovedKGroupedStream extends KGroupedStream { * @see #count(Named, Materialized) */ ImprovedKTable count(Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable reduce(Reducer reducer); @@ -78,7 +78,7 @@ ImprovedKTable count(Named named, * @see #reduce(Reducer, Materialized) */ ImprovedKTable reduce(Reducer reducer, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable reduce(Reducer reducer, Named named, @@ -88,7 +88,7 @@ ImprovedKTable reduce(Reducer reducer, Named named, * @see #reduce(Reducer, Named, Materialized) */ ImprovedKTable reduce(Reducer reducer, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator); @@ -101,7 +101,7 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, @@ -111,7 +111,7 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, - Named named, ConfiguredMaterialized> materialized); + Named named, AutoMaterialized> materialized); @Override ImprovedTimeWindowedKStream windowedBy(Windows windows); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java index 04399002..99ffb878 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -65,7 +65,7 @@ public ImprovedKTable count(final Materialized count( - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -77,7 +77,7 @@ public ImprovedKTable count(final Named named, @Override public ImprovedKTable count(final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -94,7 +94,7 @@ public ImprovedKTable reduce(final Reducer reducer, @Override public ImprovedKTable reduce(final Reducer reducer, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @@ -106,7 +106,7 @@ public ImprovedKTable reduce(final Reducer reducer, final Named named, @Override public ImprovedKTable reduce(final Reducer reducer, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } @@ -126,7 +126,7 @@ public ImprovedKTable aggregate(final Initializer initializer, @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); } @@ -140,7 +140,7 @@ public ImprovedKTable aggregate(final Initializer initializer, @Override public ImprovedKTable aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java index fe2b9788..5674c201 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java @@ -47,7 +47,7 @@ public interface ImprovedKGroupedTable extends KGroupedTable { /** * @see #count(Materialized) */ - ImprovedKTable count(ConfiguredMaterialized> materialized); + ImprovedKTable count(AutoMaterialized> materialized); @Override ImprovedKTable count(Named named, Materialized> materialized); @@ -56,7 +56,7 @@ public interface ImprovedKGroupedTable extends KGroupedTable { * @see #count(Named, Materialized) */ ImprovedKTable count(Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable count(); @@ -72,7 +72,7 @@ ImprovedKTable reduce(Reducer adder, Reducer subtractor, * @see #reduce(Reducer, Reducer, Materialized) */ ImprovedKTable reduce(Reducer adder, Reducer subtractor, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, @@ -82,7 +82,7 @@ ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named * @see #reduce(Reducer, Reducer, Named, Materialized) */ ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable reduce(Reducer adder, Reducer subtractor); @@ -97,7 +97,7 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator ImprovedKTable aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable aggregate(Initializer initializer, Aggregator adder, @@ -109,7 +109,7 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator ImprovedKTable aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable aggregate(Initializer initializer, Aggregator adder, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java index 4fa7b97d..a6304291 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java @@ -48,7 +48,7 @@ public ImprovedKTable count(final Materialized count( - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -60,7 +60,7 @@ public ImprovedKTable count(final Named named, @Override public ImprovedKTable count(final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -82,7 +82,7 @@ public ImprovedKTable reduce(final Reducer adder, final Reducer subt @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(adder, subtractor, materialized.configure(this.context.getConfigurator())); } @@ -94,7 +94,7 @@ public ImprovedKTable reduce(final Reducer adder, final Reducer subt @Override public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(adder, subtractor, named, materialized.configure(this.context.getConfigurator())); } @@ -115,7 +115,7 @@ public ImprovedKTable aggregate(final Initializer initializer, public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, adder, subtractor, materialized.configure(this.context.getConfigurator())); } @@ -131,7 +131,7 @@ public ImprovedKTable aggregate(final Initializer initializer, public ImprovedKTable aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, adder, subtractor, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index bd229d5b..2e6d029c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -244,35 +244,35 @@ KErrorStream flatMapValuesCapturingErrors( /** * @see #repartition(Repartitioned) */ - ImprovedKStream repartition(ConfiguredRepartitioned repartitioned); + ImprovedKStream repartition(AutoRepartitioned repartitioned); /** * @see #to(String, Produced) */ - void to(String topic, ConfiguredProduced produced); + void to(String topic, AutoProduced produced); /** * @see #to(TopicNameExtractor, Produced) */ - void to(TopicNameExtractor topicExtractor, ConfiguredProduced produced); + void to(TopicNameExtractor topicExtractor, AutoProduced produced); void toOutputTopic(); void toOutputTopic(Produced produced); - void toOutputTopic(ConfiguredProduced produced); + void toOutputTopic(AutoProduced produced); void toOutputTopic(String label); void toOutputTopic(String label, Produced produced); - void toOutputTopic(String label, ConfiguredProduced produced); + void toOutputTopic(String label, AutoProduced produced); void toErrorTopic(); void toErrorTopic(Produced produced); - void toErrorTopic(ConfiguredProduced produced); + void toErrorTopic(AutoProduced produced); @Override ImprovedKTable toTable(); @@ -286,7 +286,7 @@ KErrorStream flatMapValuesCapturingErrors( /** * @see #toTable(Materialized) */ - ImprovedKTable toTable(ConfiguredMaterialized> materialized); + ImprovedKTable toTable(AutoMaterialized> materialized); @Override ImprovedKTable toTable(Named named, Materialized> materialized); @@ -294,7 +294,7 @@ KErrorStream flatMapValuesCapturingErrors( /** * @see #toTable(Named, Materialized) */ - ImprovedKTable toTable(Named named, ConfiguredMaterialized> materialized); + ImprovedKTable toTable(Named named, AutoMaterialized> materialized); @Override ImprovedKGroupedStream groupBy(KeyValueMapper keySelector); @@ -307,7 +307,7 @@ ImprovedKGroupedStream groupBy(KeyValueMapper ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, - ConfiguredGrouped grouped); + AutoGrouped grouped); @Override ImprovedKGroupedStream groupByKey(); @@ -318,7 +318,7 @@ ImprovedKGroupedStream groupBy(KeyValueMapper groupByKey(ConfiguredGrouped grouped); + ImprovedKGroupedStream groupByKey(AutoGrouped grouped); @Override ImprovedKStream join(KStream otherStream, @@ -339,7 +339,7 @@ ImprovedKStream join(KStream otherStream, */ ImprovedKStream join(KStream otherStream, ValueJoiner joiner, - JoinWindows windows, ConfiguredStreamJoined streamJoined); + JoinWindows windows, AutoStreamJoined streamJoined); @Override ImprovedKStream join(KStream otherStream, @@ -351,7 +351,7 @@ ImprovedKStream join(KStream otherStream, */ ImprovedKStream join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, - ConfiguredStreamJoined streamJoined); + AutoStreamJoined streamJoined); @Override ImprovedKStream leftJoin(KStream otherStream, @@ -371,7 +371,7 @@ ImprovedKStream leftJoin(KStream otherStream, */ ImprovedKStream leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, - ConfiguredStreamJoined streamJoined); + AutoStreamJoined streamJoined); @Override ImprovedKStream leftJoin(KStream otherStream, @@ -383,7 +383,7 @@ ImprovedKStream leftJoin(KStream otherStream, */ ImprovedKStream leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, - ConfiguredStreamJoined streamJoined); + AutoStreamJoined streamJoined); @Override ImprovedKStream outerJoin(KStream otherStream, @@ -403,7 +403,7 @@ ImprovedKStream outerJoin(KStream otherStream, */ ImprovedKStream outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, - ConfiguredStreamJoined streamJoined); + AutoStreamJoined streamJoined); @Override ImprovedKStream outerJoin(KStream otherStream, @@ -415,7 +415,7 @@ ImprovedKStream outerJoin(KStream otherStream, */ ImprovedKStream outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, - ConfiguredStreamJoined streamJoined); + AutoStreamJoined streamJoined); @Override ImprovedKStream join(KTable table, ValueJoiner joiner); @@ -432,7 +432,7 @@ ImprovedKStream join(KTable table, ValueJoiner ImprovedKStream join(KTable table, ValueJoiner joiner, - ConfiguredJoined joined); + AutoJoined joined); @Override ImprovedKStream join(KTable table, @@ -443,7 +443,7 @@ ImprovedKStream join(KTable table, */ ImprovedKStream join(KTable table, ValueJoinerWithKey joiner, - ConfiguredJoined joined); + AutoJoined joined); @Override ImprovedKStream leftJoin(KTable table, @@ -463,7 +463,7 @@ ImprovedKStream leftJoin(KTable table, */ ImprovedKStream leftJoin(KTable table, ValueJoiner joiner, - ConfiguredJoined joined); + AutoJoined joined); @Override ImprovedKStream leftJoin(KTable table, @@ -474,7 +474,7 @@ ImprovedKStream leftJoin(KTable table, */ ImprovedKStream leftJoin(KTable table, ValueJoinerWithKey joiner, - ConfiguredJoined joined); + AutoJoined joined); @Override ImprovedKStream join(GlobalKTable globalTable, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 478e0bd6..0b78961e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -426,7 +426,7 @@ public ImprovedKStream repartition(final Repartitioned repartitioned } @Override - public ImprovedKStream repartition(final ConfiguredRepartitioned repartitioned) { + public ImprovedKStream repartition(final AutoRepartitioned repartitioned) { return this.repartition(repartitioned.configure(this.context.getConfigurator())); } @@ -441,7 +441,7 @@ public void to(final String topic, final Produced produced) { } @Override - public void to(final String topic, final ConfiguredProduced produced) { + public void to(final String topic, final AutoProduced produced) { this.to(topic, produced.configure(this.context.getConfigurator())); } @@ -456,7 +456,7 @@ public void to(final TopicNameExtractor topicExtractor, final Produced topicExtractor, final ConfiguredProduced produced) { + public void to(final TopicNameExtractor topicExtractor, final AutoProduced produced) { this.to(topicExtractor, produced.configure(this.context.getConfigurator())); } @@ -471,7 +471,7 @@ public void toOutputTopic(final Produced produced) { } @Override - public void toOutputTopic(final ConfiguredProduced produced) { + public void toOutputTopic(final AutoProduced produced) { this.toOutputTopic(produced.configure(this.context.getConfigurator())); } @@ -486,7 +486,7 @@ public void toOutputTopic(final String label, final Produced produced) { } @Override - public void toOutputTopic(final String label, final ConfiguredProduced produced) { + public void toOutputTopic(final String label, final AutoProduced produced) { this.toOutputTopic(label, produced.configure(this.context.getConfigurator())); } @@ -501,7 +501,7 @@ public void toErrorTopic(final Produced produced) { } @Override - public void toErrorTopic(final ConfiguredProduced produced) { + public void toErrorTopic(final AutoProduced produced) { this.toErrorTopic(produced.configure(this.context.getConfigurator())); } @@ -521,7 +521,7 @@ public ImprovedKTable toTable(final Materialized toTable(final ConfiguredMaterialized> materialized) { + public ImprovedKTable toTable(final AutoMaterialized> materialized) { return this.toTable(materialized.configure(this.context.getConfigurator())); } @@ -533,7 +533,7 @@ public ImprovedKTable toTable(final Named named, @Override public ImprovedKTable toTable(final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.toTable(named, materialized.configure(this.context.getConfigurator())); } @@ -550,7 +550,7 @@ public ImprovedKGroupedStream groupBy(final KeyValueMapper ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector, - final ConfiguredGrouped grouped) { + final AutoGrouped grouped) { return this.groupBy(keySelector, grouped.configure(this.context.getConfigurator())); } @@ -565,7 +565,7 @@ public ImprovedKGroupedStream groupByKey(final Grouped grouped) { } @Override - public ImprovedKGroupedStream groupByKey(final ConfiguredGrouped grouped) { + public ImprovedKGroupedStream groupByKey(final AutoGrouped grouped) { return this.groupByKey(grouped.configure(this.context.getConfigurator())); } @@ -595,7 +595,7 @@ public ImprovedKStream join(final KStream otherStream, @Override public ImprovedKStream join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final ConfiguredStreamJoined streamJoined) { + final AutoStreamJoined streamJoined) { return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -610,7 +610,7 @@ public ImprovedKStream join(final KStream otherStream, @Override public ImprovedKStream join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, - final ConfiguredStreamJoined streamJoined) { + final AutoStreamJoined streamJoined) { return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -640,7 +640,7 @@ public ImprovedKStream leftJoin(final KStream otherStream @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final ConfiguredStreamJoined streamJoined) { + final AutoStreamJoined streamJoined) { return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -655,7 +655,7 @@ public ImprovedKStream leftJoin(final KStream otherStream @Override public ImprovedKStream leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, - final ConfiguredStreamJoined streamJoined) { + final AutoStreamJoined streamJoined) { return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -685,7 +685,7 @@ public ImprovedKStream outerJoin(final KStream otherStrea @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final ConfiguredStreamJoined streamJoined) { + final AutoStreamJoined streamJoined) { return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -700,7 +700,7 @@ public ImprovedKStream outerJoin(final KStream otherStrea @Override public ImprovedKStream outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, - final ConfiguredStreamJoined streamJoined) { + final AutoStreamJoined streamJoined) { return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -727,7 +727,7 @@ public ImprovedKStream join(final KTable table, @Override public ImprovedKStream join(final KTable table, - final ValueJoiner joiner, final ConfiguredJoined joined) { + final ValueJoiner joiner, final AutoJoined joined) { return this.join(table, joiner, joined.configure(this.context.getConfigurator())); } @@ -742,7 +742,7 @@ public ImprovedKStream join(final KTable table, @Override public ImprovedKStream join(final KTable table, final ValueJoinerWithKey joiner, - final ConfiguredJoined joined) { + final AutoJoined joined) { return this.join(table, joiner, joined.configure(this.context.getConfigurator())); } @@ -769,7 +769,7 @@ public ImprovedKStream leftJoin(final KTable table, @Override public ImprovedKStream leftJoin(final KTable table, - final ValueJoiner joiner, final ConfiguredJoined joined) { + final ValueJoiner joiner, final AutoJoined joined) { return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); } @@ -784,7 +784,7 @@ public ImprovedKStream leftJoin(final KTable table, @Override public ImprovedKStream leftJoin(final KTable table, final ValueJoinerWithKey joiner, - final ConfiguredJoined joined) { + final AutoJoined joined) { return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java index a1280c20..d309eca4 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java @@ -63,7 +63,7 @@ ImprovedKTable filter(Predicate predicate, * @see #filter(Predicate, Materialized) */ ImprovedKTable filter(Predicate predicate, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable filter(Predicate predicate, Named named, @@ -73,7 +73,7 @@ ImprovedKTable filter(Predicate predicate, Named nam * @see #filter(Predicate, Named, Materialized) */ ImprovedKTable filter(Predicate predicate, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable filterNot(Predicate predicate); @@ -89,7 +89,7 @@ ImprovedKTable filterNot(Predicate predicate, * @see #filterNot(Predicate, Materialized) */ ImprovedKTable filterNot(Predicate predicate, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable filterNot(Predicate predicate, Named named, @@ -99,7 +99,7 @@ ImprovedKTable filterNot(Predicate predicate, Named * @see #filterNot(Predicate, Named, Materialized) */ ImprovedKTable filterNot(Predicate predicate, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable mapValues(ValueMapper mapper); @@ -121,7 +121,7 @@ ImprovedKTable mapValues(ValueMapper mapper * @see #mapValues(ValueMapper, Materialized) */ ImprovedKTable mapValues(ValueMapper mapper, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable mapValues(ValueMapper mapper, Named named, @@ -131,7 +131,7 @@ ImprovedKTable mapValues(ValueMapper mapper * @see #mapValues(ValueMapper, Named, Materialized) */ ImprovedKTable mapValues(ValueMapper mapper, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable mapValues(ValueMapperWithKey mapper, @@ -141,7 +141,7 @@ ImprovedKTable mapValues(ValueMapperWithKey ImprovedKTable mapValues(ValueMapperWithKey mapper, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, @@ -151,7 +151,7 @@ ImprovedKTable mapValues(ValueMapperWithKey ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKStream toStream(); @@ -188,7 +188,7 @@ ImprovedKTable transformValues( */ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, - ConfiguredMaterialized> materialized, String... stateStoreNames); + AutoMaterialized> materialized, String... stateStoreNames); @Override ImprovedKTable transformValues( @@ -200,7 +200,7 @@ ImprovedKTable transformValues( */ ImprovedKTable transformValues( ValueTransformerWithKeySupplier transformerSupplier, - ConfiguredMaterialized> materialized, Named named, + AutoMaterialized> materialized, Named named, String... stateStoreNames); @Override @@ -214,7 +214,7 @@ ImprovedKGroupedTable groupBy(KeyValueMapper ImprovedKGroupedTable groupBy(KeyValueMapper> selector, - ConfiguredGrouped grouped); + AutoGrouped grouped); @Override ImprovedKTable join(KTable other, ValueJoiner joiner); @@ -231,7 +231,7 @@ ImprovedKTable join(KTable other, ValueJoiner ImprovedKTable join(KTable other, ValueJoiner joiner, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable join(KTable other, ValueJoiner joiner, @@ -241,7 +241,7 @@ ImprovedKTable join(KTable other, ValueJoiner ImprovedKTable join(KTable other, ValueJoiner joiner, - Named named, ConfiguredMaterialized> materialized); + Named named, AutoMaterialized> materialized); @Override ImprovedKTable leftJoin(KTable other, @@ -262,7 +262,7 @@ ImprovedKTable leftJoin(KTable other, */ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable leftJoin(KTable other, @@ -274,7 +274,7 @@ ImprovedKTable leftJoin(KTable other, */ ImprovedKTable leftJoin(KTable other, ValueJoiner joiner, - Named named, ConfiguredMaterialized> materialized); + Named named, AutoMaterialized> materialized); @Override ImprovedKTable outerJoin(KTable other, @@ -295,7 +295,7 @@ ImprovedKTable outerJoin(KTable other, */ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable outerJoin(KTable other, @@ -307,7 +307,7 @@ ImprovedKTable outerJoin(KTable other, */ ImprovedKTable outerJoin(KTable other, ValueJoiner joiner, - Named named, ConfiguredMaterialized> materialized); + Named named, AutoMaterialized> materialized); @Override ImprovedKTable join(KTable other, Function foreignKeyExtractor, @@ -329,7 +329,7 @@ ImprovedKTable join(KTable other, Function fo * @see #join(KTable, Function, ValueJoiner, Materialized) */ ImprovedKTable join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, ConfiguredMaterialized> materialized); + ValueJoiner joiner, AutoMaterialized> materialized); @Override ImprovedKTable join(KTable other, Function foreignKeyExtractor, @@ -340,7 +340,7 @@ ImprovedKTable join(KTable other, Function fo */ ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable join(KTable other, Function foreignKeyExtractor, @@ -352,7 +352,7 @@ ImprovedKTable join(KTable other, Function fo */ ImprovedKTable join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, @@ -374,7 +374,7 @@ ImprovedKTable leftJoin(KTable other, Function ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, ConfiguredMaterialized> materialized); + ValueJoiner joiner, AutoMaterialized> materialized); @Override ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, @@ -385,7 +385,7 @@ ImprovedKTable leftJoin(KTable other, Function ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, @@ -397,5 +397,5 @@ ImprovedKTable leftJoin(KTable other, Function ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java index df88fba0..2cdac686 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java @@ -70,7 +70,7 @@ public ImprovedKTable filter(final Predicate predica @Override public ImprovedKTable filter(final Predicate predicate, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.filter(predicate, materialized.configure(this.context.getConfigurator())); } @@ -82,7 +82,7 @@ public ImprovedKTable filter(final Predicate predica @Override public ImprovedKTable filter(final Predicate predicate, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.filter(predicate, named, materialized.configure(this.context.getConfigurator())); } @@ -104,7 +104,7 @@ public ImprovedKTable filterNot(final Predicate pred @Override public ImprovedKTable filterNot(final Predicate predicate, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.filterNot(predicate, materialized.configure(this.context.getConfigurator())); } @@ -116,7 +116,7 @@ public ImprovedKTable filterNot(final Predicate pred @Override public ImprovedKTable filterNot(final Predicate predicate, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.filterNot(predicate, named, materialized.configure(this.context.getConfigurator())); } @@ -149,7 +149,7 @@ public ImprovedKTable mapValues(final ValueMapper ImprovedKTable mapValues(final ValueMapper mapper, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); } @@ -161,7 +161,7 @@ public ImprovedKTable mapValues(final ValueMapper ImprovedKTable mapValues(final ValueMapper mapper, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); } @@ -173,7 +173,7 @@ public ImprovedKTable mapValues(final ValueMapperWithKey ImprovedKTable mapValues(final ValueMapperWithKey mapper, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); } @@ -185,7 +185,7 @@ public ImprovedKTable mapValues(final ValueMapperWithKey ImprovedKTable mapValues(final ValueMapperWithKey mapper, - final Named named, final ConfiguredMaterialized> materialized) { + final Named named, final AutoMaterialized> materialized) { return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); } @@ -240,7 +240,7 @@ public ImprovedKTable transformValues( @Override public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, - final ConfiguredMaterialized> materialized, + final AutoMaterialized> materialized, final String... stateStoreNames) { return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), stateStoreNames); @@ -258,7 +258,7 @@ public ImprovedKTable transformValues( @Override public ImprovedKTable transformValues( final ValueTransformerWithKeySupplier transformerSupplier, - final ConfiguredMaterialized> materialized, final Named named, + final AutoMaterialized> materialized, final Named named, final String... stateStoreNames) { return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), named, stateStoreNames); @@ -278,7 +278,7 @@ public ImprovedKGroupedTable groupBy( @Override public ImprovedKGroupedTable groupBy( - final KeyValueMapper> selector, final ConfiguredGrouped grouped) { + final KeyValueMapper> selector, final AutoGrouped grouped) { return this.groupBy(selector, grouped.configure(this.context.getConfigurator())); } @@ -307,7 +307,7 @@ public ImprovedKTable join(final KTable other, @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.join(other, joiner, materialized.configure(this.context.getConfigurator())); } @@ -322,7 +322,7 @@ public ImprovedKTable join(final KTable other, @Override public ImprovedKTable join(final KTable other, final ValueJoiner joiner, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.join(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -351,7 +351,7 @@ public ImprovedKTable leftJoin(final KTable other, @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.leftJoin(other, joiner, materialized.configure(this.context.getConfigurator())); } @@ -366,7 +366,7 @@ public ImprovedKTable leftJoin(final KTable other, @Override public ImprovedKTable leftJoin(final KTable other, final ValueJoiner joiner, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.leftJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -395,7 +395,7 @@ public ImprovedKTable outerJoin(final KTable other, @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.outerJoin(other, joiner, materialized.configure(this.context.getConfigurator())); } @@ -410,7 +410,7 @@ public ImprovedKTable outerJoin(final KTable other, @Override public ImprovedKTable outerJoin(final KTable other, final ValueJoiner joiner, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.outerJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -450,7 +450,7 @@ public ImprovedKTable join(final KTable other, public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } @@ -467,7 +467,7 @@ public ImprovedKTable join(final KTable other, public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.join(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -485,7 +485,7 @@ public ImprovedKTable join(final KTable other, public ImprovedKTable join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.join(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); } @@ -526,7 +526,7 @@ public ImprovedKTable leftJoin(final KTable other, public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } @@ -544,7 +544,7 @@ public ImprovedKTable leftJoin(final KTable other, public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -563,7 +563,7 @@ public ImprovedKTable leftJoin(final KTable other, public ImprovedKTable leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java index 22639cee..9c02b827 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java @@ -56,7 +56,7 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge * @see #aggregate(Initializer, Merger, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, @@ -66,5 +66,5 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Merge * @see #aggregate(Initializer, Merger, Named, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, - Named named, ConfiguredMaterialized> materialized); + Named named, AutoMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java index f589076e..e5e19819 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java @@ -64,7 +64,7 @@ public ImprovedKTable, V> aggregate(final Initializer initializer @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, sessionMerger, materialized.configure(this.context.getConfigurator())); } @@ -78,7 +78,7 @@ public ImprovedKTable, V> aggregate(final Initializer initializer @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Merger sessionMerger, - final Named named, final ConfiguredMaterialized> materialized) { + final Named named, final AutoMaterialized> materialized) { return this.aggregate(initializer, sessionMerger, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java index fa876a6e..935a3aa5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java @@ -56,7 +56,7 @@ public interface ImprovedSessionWindowedKStream extends SessionWindowedKSt /** * @see #count(Materialized) */ - ImprovedKTable, Long> count(ConfiguredMaterialized> materialized); + ImprovedKTable, Long> count(AutoMaterialized> materialized); @Override ImprovedKTable, Long> count(Named named, @@ -66,7 +66,7 @@ ImprovedKTable, Long> count(Named named, * @see #count(Named, Materialized) */ ImprovedKTable, Long> count(Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, VR> aggregate(Initializer initializer, @@ -89,7 +89,7 @@ ImprovedKTable, VR> aggregate(Initializer initializer, ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, VR> aggregate(Initializer initializer, @@ -103,7 +103,7 @@ ImprovedKTable, VR> aggregate(Initializer initializer, ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, V> reduce(Reducer reducer); @@ -119,7 +119,7 @@ ImprovedKTable, V> reduce(Reducer reducer, * @see #reduce(Reducer, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, V> reduce(Reducer reducer, Named named, @@ -129,7 +129,7 @@ ImprovedKTable, V> reduce(Reducer reducer, Named named, * @see #reduce(Reducer, Named, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedSessionWindowedKStream emitStrategy(EmitStrategy emitStrategy); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java index 268d722d..ecfe2646 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java @@ -62,7 +62,7 @@ public ImprovedKTable, Long> count( @Override public ImprovedKTable, Long> count( - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -74,7 +74,7 @@ public ImprovedKTable, Long> count(final Named named, @Override public ImprovedKTable, Long> count(final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -101,7 +101,7 @@ public ImprovedKTable, VR> aggregate(final Initializer init @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, sessionMerger, materialized.configure(this.context.getConfigurator())); } @@ -118,7 +118,7 @@ public ImprovedKTable, VR> aggregate(final Initializer init public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, sessionMerger, named, materialized.configure(this.context.getConfigurator())); } @@ -141,7 +141,7 @@ public ImprovedKTable, V> reduce(final Reducer reducer, @Override public ImprovedKTable, V> reduce(final Reducer reducer, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @@ -153,7 +153,7 @@ public ImprovedKTable, V> reduce(final Reducer reducer, final Nam @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java index 4bf4e000..7f21d401 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java @@ -54,7 +54,7 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, * @see #aggregate(Initializer, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, @@ -64,5 +64,5 @@ ImprovedKTable, VOut> aggregate(Initializer initializer, Named * @see #aggregate(Initializer, Named, Materialized) */ ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java index 752366b5..63982d89 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java @@ -58,7 +58,7 @@ public ImprovedKTable, V> aggregate(final Initializer initializer @Override public ImprovedKTable, V> aggregate(final Initializer initializer, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); } @@ -70,7 +70,7 @@ public ImprovedKTable, V> aggregate(final Initializer initializer @Override public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java index c2a4f665..567a6586 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java @@ -55,7 +55,7 @@ public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream, Long> count(ConfiguredMaterialized> materialized); + ImprovedKTable, Long> count(AutoMaterialized> materialized); @Override ImprovedKTable, Long> count(Named named, @@ -65,7 +65,7 @@ ImprovedKTable, Long> count(Named named, * @see #count(Named, Materialized) */ ImprovedKTable, Long> count(Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, VR> aggregate(Initializer initializer, @@ -86,7 +86,7 @@ ImprovedKTable, VR> aggregate(Initializer initializer, */ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, VR> aggregate(Initializer initializer, @@ -98,7 +98,7 @@ ImprovedKTable, VR> aggregate(Initializer initializer, */ ImprovedKTable, VR> aggregate(Initializer initializer, Aggregator aggregator, - Named named, ConfiguredMaterialized> materialized); + Named named, AutoMaterialized> materialized); @Override ImprovedKTable, V> reduce(Reducer reducer); @@ -114,7 +114,7 @@ ImprovedKTable, V> reduce(Reducer reducer, * @see #reduce(Reducer, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedKTable, V> reduce(Reducer reducer, Named named, @@ -124,7 +124,7 @@ ImprovedKTable, V> reduce(Reducer reducer, Named named, * @see #reduce(Reducer, Named, Materialized) */ ImprovedKTable, V> reduce(Reducer reducer, Named named, - ConfiguredMaterialized> materialized); + AutoMaterialized> materialized); @Override ImprovedTimeWindowedKStream emitStrategy(EmitStrategy emitStrategy); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java index 0c858628..72e6d524 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java @@ -61,7 +61,7 @@ public ImprovedKTable, Long> count( @Override public ImprovedKTable, Long> count( - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -73,7 +73,7 @@ public ImprovedKTable, Long> count(final Named named, @Override public ImprovedKTable, Long> count(final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -99,7 +99,7 @@ public ImprovedKTable, VR> aggregate(final Initializer init @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); } @@ -113,7 +113,7 @@ public ImprovedKTable, VR> aggregate(final Initializer init @Override public ImprovedKTable, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); } @@ -135,7 +135,7 @@ public ImprovedKTable, V> reduce(final Reducer reducer, @Override public ImprovedKTable, V> reduce(final Reducer reducer, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @@ -147,7 +147,7 @@ public ImprovedKTable, V> reduce(final Reducer reducer, final Nam @Override public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, - final ConfiguredMaterialized> materialized) { + final AutoMaterialized> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index d2ae9337..ea0405ec 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -66,7 +66,7 @@ public ImprovedKStream stream(final String topic, final Consumed ImprovedKStream stream(final String topic, final ConfiguredConsumed consumed) { + public ImprovedKStream stream(final String topic, final AutoConsumed consumed) { return this.stream(topic, consumed.configure(this.createConfigurator())); } @@ -88,7 +88,7 @@ public ImprovedKStream stream(final Collection topics, fina * @see StreamsBuilder#stream(Collection, Consumed) */ public ImprovedKStream stream(final Collection topics, - final ConfiguredConsumed consumed) { + final AutoConsumed consumed) { return this.stream(topics, consumed.configure(this.createConfigurator())); } @@ -109,7 +109,7 @@ public ImprovedKStream stream(final Pattern topicPattern, final Con /** * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream stream(final Pattern topicPattern, final ConfiguredConsumed consumed) { + public ImprovedKStream stream(final Pattern topicPattern, final AutoConsumed consumed) { return this.stream(topicPattern, consumed.configure(this.createConfigurator())); } @@ -133,7 +133,7 @@ public ImprovedKStream streamInput(final Consumed consumed) { * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream streamInput(final ConfiguredConsumed consumed) { + public ImprovedKStream streamInput(final AutoConsumed consumed) { return this.streamInput(consumed.configure(this.createConfigurator())); } @@ -170,7 +170,7 @@ public ImprovedKStream streamInput(final String label, final Consum * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream streamInput(final String label, final ConfiguredConsumed consumed) { + public ImprovedKStream streamInput(final String label, final AutoConsumed consumed) { return this.streamInput(label, consumed.configure(this.createConfigurator())); } @@ -206,7 +206,7 @@ public ImprovedKStream streamInputPattern(final Consumed cons * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream streamInputPattern(final ConfiguredConsumed consumed) { + public ImprovedKStream streamInputPattern(final AutoConsumed consumed) { return this.streamInputPattern(consumed.configure(this.createConfigurator())); } @@ -244,7 +244,7 @@ public ImprovedKStream streamInputPattern(final String label, final * @see StreamsBuilder#stream(Pattern, Consumed) */ public ImprovedKStream streamInputPattern(final String label, - final ConfiguredConsumed consumed) { + final AutoConsumed consumed) { return this.streamInputPattern(label, consumed.configure(this.createConfigurator())); } @@ -287,10 +287,10 @@ public StreamsContext getContext() { /** * Create stores using application context to lazily configures Serdes - * @return {@code ConfiguredStores} + * @return {@code AutoStores} */ - public ConfiguredStores stores() { - return new ConfiguredStores(this.createConfigurator()); + public AutoStores stores() { + return new AutoStores(this.createConfigurator()); } Topology build() { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java index d7f4487b..f3998db5 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java @@ -65,15 +65,15 @@ void shouldReduceUsingMaterialized() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKGroupedStream grouped = input.groupByKey( - ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKTable reduced = grouped.reduce(Long::sum, - ConfiguredMaterialized.with(Preconfigured.create(Serdes.Long()), + AutoMaterialized.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - reduced.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + reduced.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -131,16 +131,16 @@ void shouldAggregateUsingMaterialized() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKGroupedStream grouped = input.groupByKey( - ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKTable aggregated = grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, - ConfiguredMaterialized.with(Preconfigured.create(Serdes.Long()), + AutoMaterialized.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - aggregated.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + aggregated.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index a985359f..73a89444 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -95,9 +95,9 @@ void shouldWriteToOutputUsingProduced() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.toOutputTopic(ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.toOutputTopic(AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -147,9 +147,9 @@ void shouldWriteToLabeledOutputUsingProduced() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.toOutputTopic("label", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.toOutputTopic("label", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -199,9 +199,9 @@ void shouldWriteToErrorUsingProduced() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.toErrorTopic(ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.toErrorTopic(AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -844,12 +844,12 @@ void shouldRepartitionUsingRepartitioned() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream repartitioned = input.repartition( - ConfiguredRepartitioned.with(Preconfigured.create(Serdes.Long()), + AutoRepartitioned.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - repartitioned.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + repartitioned.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -897,12 +897,12 @@ void shouldConvertToTableUsingMaterialized() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKTable table = input.toTable( - ConfiguredMaterialized.with(Preconfigured.create(Serdes.Long()), + AutoMaterialized.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - table.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + table.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1118,17 +1118,17 @@ void shouldJoinUsingJoined() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream otherInput = builder.stream("other_input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream joined = input.join(otherInput, Long::sum, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1158,17 +1158,17 @@ void shouldJoinWithKeyUsingJoined() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream otherInput = builder.stream("other_input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream joined = input.join(otherInput, (k, v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1198,17 +1198,17 @@ void shouldLeftJoinUsingJoined() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream otherInput = builder.stream("other_input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1247,17 +1247,17 @@ void shouldLeftJoinWithKeyUsingJoined() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream otherInput = builder.stream("other_input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream joined = input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1296,17 +1296,17 @@ void shouldOuterJoinUsingJoined() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream otherInput = builder.stream("other_input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1345,17 +1345,17 @@ void shouldOuterJoinWithKeyUsingJoined() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream otherInput = builder.stream("other_input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKStream joined = input.outerJoin(otherInput, (k, v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - ConfiguredStreamJoined.with(Preconfigured.create(Serdes.Long()), + AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1422,13 +1422,13 @@ void shouldGroupByKeyUsingGrouped() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKGroupedStream grouped = input.groupByKey( - ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKTable count = grouped.count(); - count.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + count.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -1486,13 +1486,13 @@ void shouldGroupByUsingGrouped() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKGroupedStream grouped = input.groupBy((k, v) -> v, - ConfiguredGrouped.with(Preconfigured.create(Serdes.Long()), + AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final ImprovedKTable count = grouped.count(); - count.toStream().to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + count.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index 99db1a3d..a4eb2662 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -63,9 +63,9 @@ void shouldReadFromInputUsingConsumed() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.streamInput( - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -115,9 +115,9 @@ void shouldReadFromLabeledInputUsingConsumed() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.streamInput("label", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -167,9 +167,9 @@ void shouldReadFromPatternInputUsingConsumed() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.streamInputPattern( - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; @@ -219,9 +219,9 @@ void shouldReadFromLabeledPatternInputUsingConsumed() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.streamInputPattern("label", - ConfiguredConsumed.with(Preconfigured.create(Serdes.Long()), + AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - input.to("output", ConfiguredProduced.with(Preconfigured.create(Serdes.Long()), + input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } }; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java index 90f9de6f..890ae563 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java @@ -24,8 +24,8 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ConfiguredConsumed; -import com.bakdata.kafka.ConfiguredProduced; +import com.bakdata.kafka.AutoConsumed; +import com.bakdata.kafka.AutoProduced; import com.bakdata.kafka.ImprovedKStream; import com.bakdata.kafka.Preconfigured; import com.bakdata.kafka.SerdeConfig; @@ -54,8 +54,8 @@ public void buildTopology(final TopologyBuilder builder) { final Preconfigured> valueSerde = newValueSerde(); final Preconfigured> keySerde = newKeySerde(); final ImprovedKStream input = - builder.streamInput(ConfiguredConsumed.with(keySerde, valueSerde)); - input.toOutputTopic(ConfiguredProduced.with(keySerde, valueSerde)); + builder.streamInput(AutoConsumed.with(keySerde, valueSerde)); + input.toOutputTopic(AutoProduced.with(keySerde, valueSerde)); } @Override From b6903cc3b462d2b1ac2189bdc8eb55db8d3a0d33 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 16:51:26 +0100 Subject: [PATCH 28/72] Rename --- ...provedCogroupedKStream.java => CogroupedKStreamX.java} | 4 ++-- ...CogroupedStreamImpl.java => CogroupedStreamXImpl.java} | 4 ++-- .../java/com/bakdata/kafka/ImprovedKGroupedStream.java | 2 +- .../com/bakdata/kafka/ImprovedKGroupedStreamImpl.java | 2 +- .../src/main/java/com/bakdata/kafka/StreamsContext.java | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedCogroupedKStream.java => CogroupedKStreamX.java} (95%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedCogroupedStreamImpl.java => CogroupedStreamXImpl.java} (95%) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java similarity index 95% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java index bab48523..f18d99a2 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java @@ -43,10 +43,10 @@ * @param type of keys * @param type of values */ -public interface ImprovedCogroupedKStream extends CogroupedKStream { +public interface CogroupedKStreamX extends CogroupedKStream { @Override - ImprovedCogroupedKStream cogroup(KGroupedStream groupedStream, + CogroupedKStreamX cogroup(KGroupedStream groupedStream, Aggregator aggregator); @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java similarity index 95% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java index 5e709e3d..30165604 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java @@ -40,13 +40,13 @@ import org.apache.kafka.streams.state.KeyValueStore; @RequiredArgsConstructor -class ImprovedCogroupedStreamImpl implements ImprovedCogroupedKStream { +class CogroupedStreamXImpl implements CogroupedKStreamX { private final @NonNull CogroupedKStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedCogroupedKStream cogroup(final KGroupedStream groupedStream, + public CogroupedKStreamX cogroup(final KGroupedStream groupedStream, final Aggregator aggregator) { final KGroupedStream other = StreamsContext.maybeUnwrap(groupedStream); return this.context.wrap(this.wrapped.cogroup(other, aggregator)); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java index 88aa129c..51586c7f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java @@ -123,5 +123,5 @@ ImprovedKTable aggregate(Initializer initializer, Aggregator windowedBy(SessionWindows windows); @Override - ImprovedCogroupedKStream cogroup(Aggregator aggregator); + CogroupedKStreamX cogroup(Aggregator aggregator); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java index 99ffb878..aff6ad88 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java @@ -160,7 +160,7 @@ public ImprovedSessionWindowedKStream windowedBy(final SessionWindows wind } @Override - public ImprovedCogroupedKStream cogroup(final Aggregator aggregator) { + public CogroupedKStreamX cogroup(final Aggregator aggregator) { return this.context.wrap(this.wrapped.cogroup(aggregator)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index 97acd1de..9417de42 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -175,11 +175,11 @@ public ImprovedSessionWindowedCogroupedKStream wrap( * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedCogroupedKStream wrap(final CogroupedKStream stream) { - if (stream instanceof ImprovedCogroupedKStream) { - return (ImprovedCogroupedKStream) stream; + public CogroupedKStreamX wrap(final CogroupedKStream stream) { + if (stream instanceof CogroupedKStreamX) { + return (CogroupedKStreamX) stream; } - return new ImprovedCogroupedStreamImpl<>(stream, this); + return new CogroupedStreamXImpl<>(stream, this); } /** From 0088a5796e838af95f635f520cea66c2fc154848 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 17:28:16 +0100 Subject: [PATCH 29/72] Rename --- .../kafka/test_applications/WordCount.java | 4 +- ...chedKStream.java => BranchedKStreamX.java} | 6 +- ...eamImpl.java => BranchedKStreamXImpl.java} | 6 +- .../com/bakdata/kafka/CogroupedKStreamX.java | 18 +-- .../bakdata/kafka/CogroupedStreamXImpl.java | 18 +-- .../com/bakdata/kafka/ImprovedKStream.java | 28 ++-- .../bakdata/kafka/ImprovedKStreamImpl.java | 28 ++-- ...roupedStream.java => KGroupedStreamX.java} | 40 ++--- ...reamImpl.java => KGroupedStreamXImpl.java} | 40 ++--- ...KGroupedTable.java => KGroupedTableX.java} | 36 ++--- ...TableImpl.java => KGroupedTableXImpl.java} | 36 ++--- .../{ImprovedKTable.java => KTableX.java} | 142 +++++++++--------- ...provedKTableImpl.java => KTableXImpl.java} | 142 +++++++++--------- ... => SessionWindowedCogroupedKStreamX.java} | 14 +- ... SessionWindowedCogroupedStreamXImpl.java} | 14 +- ...ream.java => SessionWindowedKStreamX.java} | 40 ++--- ...l.java => SessionWindowedStreamXImpl.java} | 40 ++--- .../com/bakdata/kafka/StreamsContext.java | 112 +++++++------- ...ava => TimeWindowedCogroupedKStreamX.java} | 14 +- ... => TimeWindowedCogroupedStreamXImpl.java} | 14 +- ...KStream.java => TimeWindowedKStreamX.java} | 40 ++--- ...Impl.java => TimeWindowedStreamXImpl.java} | 40 ++--- .../bakdata/kafka/ImprovedKStreamTest.java | 20 +-- ...reamTest.java => KGroupedStreamXTest.java} | 18 +-- .../ComplexTopologyApplication.java | 4 +- .../kafka/test_applications/WordCount.java | 4 +- .../test_applications/WordCountPattern.java | 4 +- 27 files changed, 461 insertions(+), 461 deletions(-) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedBranchedKStream.java => BranchedKStreamX.java} (87%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedBranchedKStreamImpl.java => BranchedKStreamXImpl.java} (90%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKGroupedStream.java => KGroupedStreamX.java} (66%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKGroupedStreamImpl.java => KGroupedStreamXImpl.java} (78%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKGroupedTable.java => KGroupedTableX.java} (68%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKGroupedTableImpl.java => KGroupedTableXImpl.java} (78%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKTable.java => KTableX.java} (64%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKTableImpl.java => KTableXImpl.java} (78%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedSessionWindowedCogroupedKStream.java => SessionWindowedCogroupedKStreamX.java} (74%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedSessionWindowedCogroupedStreamImpl.java => SessionWindowedCogroupedStreamXImpl.java} (83%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedSessionWindowedKStream.java => SessionWindowedKStreamX.java} (72%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedSessionWindowedStreamImpl.java => SessionWindowedStreamXImpl.java} (78%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedTimeWindowedCogroupedKStream.java => TimeWindowedCogroupedKStreamX.java} (78%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedTimeWindowedCogroupedStreamImpl.java => TimeWindowedCogroupedStreamXImpl.java} (79%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedTimeWindowedKStream.java => TimeWindowedKStreamX.java} (70%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedTimeWindowedStreamImpl.java => TimeWindowedStreamXImpl.java} (76%) rename streams-bootstrap-core/src/test/java/com/bakdata/kafka/{ImprovedKGroupedStreamTest.java => KGroupedStreamXTest.java} (90%) diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java index 3418860f..e4f82f20 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java @@ -25,7 +25,7 @@ package com.bakdata.kafka.test_applications; import com.bakdata.kafka.ImprovedKStream; -import com.bakdata.kafka.ImprovedKTable; +import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -46,7 +46,7 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream textLines = builder.streamInput(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); - final ImprovedKTable wordCounts = textLines + final KTableX wordCounts = textLines .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) .groupBy((key, word) -> word) .count(Materialized.as("counts")); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java similarity index 87% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java index cacc1c3e..6a0d75b5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java @@ -36,13 +36,13 @@ * @param type of keys * @param type of values */ -public interface ImprovedBranchedKStream extends BranchedKStream { +public interface BranchedKStreamX extends BranchedKStream { @Override - ImprovedBranchedKStream branch(Predicate predicate); + BranchedKStreamX branch(Predicate predicate); @Override - ImprovedBranchedKStream branch(Predicate predicate, Branched branched); + BranchedKStreamX branch(Predicate predicate, Branched branched); @Override Map> defaultBranch(); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java similarity index 90% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java index 2f390a03..7a005540 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedBranchedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java @@ -35,18 +35,18 @@ import org.apache.kafka.streams.kstream.Predicate; @RequiredArgsConstructor -class ImprovedBranchedKStreamImpl implements ImprovedBranchedKStream { +class BranchedKStreamXImpl implements BranchedKStreamX { private final @NonNull BranchedKStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedBranchedKStream branch(final Predicate predicate) { + public BranchedKStreamX branch(final Predicate predicate) { return this.context.wrap(this.wrapped.branch(predicate)); } @Override - public ImprovedBranchedKStream branch(final Predicate predicate, + public BranchedKStreamX branch(final Predicate predicate, final Branched branched) { return this.context.wrap(this.wrapped.branch(predicate, branched)); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java index f18d99a2..9ca502d5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java @@ -50,37 +50,37 @@ CogroupedKStreamX cogroup(KGroupedStream groupedStream, Aggregator aggregator); @Override - ImprovedKTable aggregate(Initializer initializer); + KTableX aggregate(Initializer initializer); @Override - ImprovedKTable aggregate(Initializer initializer, Named named); + KTableX aggregate(Initializer initializer, Named named); @Override - ImprovedKTable aggregate(Initializer initializer, + KTableX aggregate(Initializer initializer, Materialized> materialized); /** * @see #aggregate(Initializer, Materialized) */ - ImprovedKTable aggregate(Initializer initializer, + KTableX aggregate(Initializer initializer, AutoMaterialized> materialized); @Override - ImprovedKTable aggregate(Initializer initializer, Named named, + KTableX aggregate(Initializer initializer, Named named, Materialized> materialized); /** * @see #aggregate(Initializer, Named, Materialized) */ - ImprovedKTable aggregate(Initializer initializer, Named named, + KTableX aggregate(Initializer initializer, Named named, AutoMaterialized> materialized); @Override - ImprovedTimeWindowedCogroupedKStream windowedBy(Windows windows); + TimeWindowedCogroupedKStreamX windowedBy(Windows windows); @Override - ImprovedTimeWindowedCogroupedKStream windowedBy(SlidingWindows windows); + TimeWindowedCogroupedKStreamX windowedBy(SlidingWindows windows); @Override - ImprovedSessionWindowedCogroupedKStream windowedBy(SessionWindows windows); + SessionWindowedCogroupedKStreamX windowedBy(SessionWindows windows); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java index 30165604..055dfa17 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java @@ -53,51 +53,51 @@ public CogroupedKStreamX cogroup(final KGroupedStream groupe } @Override - public ImprovedKTable aggregate(final Initializer initializer) { + public KTableX aggregate(final Initializer initializer) { return this.context.wrap(this.wrapped.aggregate(initializer)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, final Named named) { + public KTableX aggregate(final Initializer initializer, final Named named) { return this.context.wrap(this.wrapped.aggregate(initializer, named)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, materialized)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final AutoMaterialized> materialized) { return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable aggregate(final Initializer initializer, final Named named, + public KTableX aggregate(final Initializer initializer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, named, materialized)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, final Named named, + public KTableX aggregate(final Initializer initializer, final Named named, final AutoMaterialized> materialized) { return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedTimeWindowedCogroupedKStream windowedBy(final Windows windows) { + public TimeWindowedCogroupedKStreamX windowedBy(final Windows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override - public ImprovedTimeWindowedCogroupedKStream windowedBy(final SlidingWindows windows) { + public TimeWindowedCogroupedKStreamX windowedBy(final SlidingWindows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override - public ImprovedSessionWindowedCogroupedKStream windowedBy(final SessionWindows windows) { + public SessionWindowedCogroupedKStreamX windowedBy(final SessionWindows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 2e6d029c..7ec8aadb 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -218,10 +218,10 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKStream[] branch(Predicate... predicates); @Override - ImprovedBranchedKStream split(); + BranchedKStreamX split(); @Override - ImprovedBranchedKStream split(Named named); + BranchedKStreamX split(Named named); @Override ImprovedKStream merge(KStream stream); @@ -275,50 +275,50 @@ KErrorStream flatMapValuesCapturingErrors( void toErrorTopic(AutoProduced produced); @Override - ImprovedKTable toTable(); + KTableX toTable(); @Override - ImprovedKTable toTable(Named named); + KTableX toTable(Named named); @Override - ImprovedKTable toTable(Materialized> materialized); + KTableX toTable(Materialized> materialized); /** * @see #toTable(Materialized) */ - ImprovedKTable toTable(AutoMaterialized> materialized); + KTableX toTable(AutoMaterialized> materialized); @Override - ImprovedKTable toTable(Named named, Materialized> materialized); + KTableX toTable(Named named, Materialized> materialized); /** * @see #toTable(Named, Materialized) */ - ImprovedKTable toTable(Named named, AutoMaterialized> materialized); + KTableX toTable(Named named, AutoMaterialized> materialized); @Override - ImprovedKGroupedStream groupBy(KeyValueMapper keySelector); + KGroupedStreamX groupBy(KeyValueMapper keySelector); @Override - ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, + KGroupedStreamX groupBy(KeyValueMapper keySelector, Grouped grouped); /** * @see #groupBy(KeyValueMapper, Grouped) */ - ImprovedKGroupedStream groupBy(KeyValueMapper keySelector, + KGroupedStreamX groupBy(KeyValueMapper keySelector, AutoGrouped grouped); @Override - ImprovedKGroupedStream groupByKey(); + KGroupedStreamX groupByKey(); @Override - ImprovedKGroupedStream groupByKey(Grouped grouped); + KGroupedStreamX groupByKey(Grouped grouped); /** * @see #groupByKey(Grouped) */ - ImprovedKGroupedStream groupByKey(AutoGrouped grouped); + KGroupedStreamX groupByKey(AutoGrouped grouped); @Override ImprovedKStream join(KStream otherStream, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java index 0b78961e..4d8a090c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java @@ -384,12 +384,12 @@ public ImprovedKStream[] branch(final Named named, final Predicate split() { + public BranchedKStreamX split() { return this.context.wrap(this.wrapped.split()); } @Override - public ImprovedBranchedKStream split(final Named named) { + public BranchedKStreamX split(final Named named) { return this.context.wrap(this.wrapped.split(named)); } @@ -506,66 +506,66 @@ public void toErrorTopic(final AutoProduced produced) { } @Override - public ImprovedKTable toTable() { + public KTableX toTable() { return this.context.wrap(this.wrapped.toTable()); } @Override - public ImprovedKTable toTable(final Named named) { + public KTableX toTable(final Named named) { return this.context.wrap(this.wrapped.toTable(named)); } @Override - public ImprovedKTable toTable(final Materialized> materialized) { + public KTableX toTable(final Materialized> materialized) { return this.context.wrap(this.wrapped.toTable(materialized)); } @Override - public ImprovedKTable toTable(final AutoMaterialized> materialized) { + public KTableX toTable(final AutoMaterialized> materialized) { return this.toTable(materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable toTable(final Named named, + public KTableX toTable(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.toTable(named, materialized)); } @Override - public ImprovedKTable toTable(final Named named, + public KTableX toTable(final Named named, final AutoMaterialized> materialized) { return this.toTable(named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector) { + public KGroupedStreamX groupBy(final KeyValueMapper keySelector) { return this.context.wrap(this.wrapped.groupBy(keySelector)); } @Override - public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector, + public KGroupedStreamX groupBy(final KeyValueMapper keySelector, final Grouped grouped) { return this.context.wrap(this.wrapped.groupBy(keySelector, grouped)); } @Override - public ImprovedKGroupedStream groupBy(final KeyValueMapper keySelector, + public KGroupedStreamX groupBy(final KeyValueMapper keySelector, final AutoGrouped grouped) { return this.groupBy(keySelector, grouped.configure(this.context.getConfigurator())); } @Override - public ImprovedKGroupedStream groupByKey() { + public KGroupedStreamX groupByKey() { return this.context.wrap(this.wrapped.groupByKey()); } @Override - public ImprovedKGroupedStream groupByKey(final Grouped grouped) { + public KGroupedStreamX groupByKey(final Grouped grouped) { return this.context.wrap(this.wrapped.groupByKey(grouped)); } @Override - public ImprovedKGroupedStream groupByKey(final AutoGrouped grouped) { + public KGroupedStreamX groupByKey(final AutoGrouped grouped) { return this.groupByKey(grouped.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java similarity index 66% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java index 51586c7f..f4f8c70a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java @@ -43,84 +43,84 @@ * @param type of keys * @param type of values */ -public interface ImprovedKGroupedStream extends KGroupedStream { +public interface KGroupedStreamX extends KGroupedStream { @Override - ImprovedKTable count(); + KTableX count(); @Override - ImprovedKTable count(Named named); + KTableX count(Named named); @Override - ImprovedKTable count(Materialized> materialized); + KTableX count(Materialized> materialized); /** * @see #count(Materialized) */ - ImprovedKTable count(AutoMaterialized> materialized); + KTableX count(AutoMaterialized> materialized); @Override - ImprovedKTable count(Named named, Materialized> materialized); + KTableX count(Named named, Materialized> materialized); /** * @see #count(Named, Materialized) */ - ImprovedKTable count(Named named, + KTableX count(Named named, AutoMaterialized> materialized); @Override - ImprovedKTable reduce(Reducer reducer); + KTableX reduce(Reducer reducer); @Override - ImprovedKTable reduce(Reducer reducer, Materialized> materialized); + KTableX reduce(Reducer reducer, Materialized> materialized); /** * @see #reduce(Reducer, Materialized) */ - ImprovedKTable reduce(Reducer reducer, + KTableX reduce(Reducer reducer, AutoMaterialized> materialized); @Override - ImprovedKTable reduce(Reducer reducer, Named named, + KTableX reduce(Reducer reducer, Named named, Materialized> materialized); /** * @see #reduce(Reducer, Named, Materialized) */ - ImprovedKTable reduce(Reducer reducer, Named named, + KTableX reduce(Reducer reducer, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator); + KTableX aggregate(Initializer initializer, Aggregator aggregator); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + KTableX aggregate(Initializer initializer, Aggregator aggregator, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Materialized) */ - ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + KTableX aggregate(Initializer initializer, Aggregator aggregator, AutoMaterialized> materialized); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + KTableX aggregate(Initializer initializer, Aggregator aggregator, Named named, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Named, Materialized) */ - ImprovedKTable aggregate(Initializer initializer, Aggregator aggregator, + KTableX aggregate(Initializer initializer, Aggregator aggregator, Named named, AutoMaterialized> materialized); @Override - ImprovedTimeWindowedKStream windowedBy(Windows windows); + TimeWindowedKStreamX windowedBy(Windows windows); @Override - ImprovedTimeWindowedKStream windowedBy(SlidingWindows windows); + TimeWindowedKStreamX windowedBy(SlidingWindows windows); @Override - ImprovedSessionWindowedKStream windowedBy(SessionWindows windows); + SessionWindowedKStreamX windowedBy(SessionWindows windows); @Override CogroupedKStreamX cogroup(Aggregator aggregator); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java similarity index 78% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java index aff6ad88..da954f02 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java @@ -42,120 +42,120 @@ import org.apache.kafka.streams.state.KeyValueStore; @RequiredArgsConstructor -class ImprovedKGroupedStreamImpl implements ImprovedKGroupedStream { +class KGroupedStreamXImpl implements KGroupedStreamX { @Getter(AccessLevel.PACKAGE) private final @NonNull KGroupedStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable count() { + public KTableX count() { return this.context.wrap(this.wrapped.count()); } @Override - public ImprovedKTable count(final Named named) { + public KTableX count(final Named named) { return this.context.wrap(this.wrapped.count(named)); } @Override - public ImprovedKTable count(final Materialized> materialized) { + public KTableX count(final Materialized> materialized) { return this.context.wrap(this.wrapped.count(materialized)); } @Override - public ImprovedKTable count( + public KTableX count( final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable count(final Named named, + public KTableX count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } @Override - public ImprovedKTable count(final Named named, + public KTableX count(final Named named, final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable reduce(final Reducer reducer) { + public KTableX reduce(final Reducer reducer) { return this.context.wrap(this.wrapped.reduce(reducer)); } @Override - public ImprovedKTable reduce(final Reducer reducer, + public KTableX reduce(final Reducer reducer, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } @Override - public ImprovedKTable reduce(final Reducer reducer, + public KTableX reduce(final Reducer reducer, final AutoMaterialized> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable reduce(final Reducer reducer, final Named named, + public KTableX reduce(final Reducer reducer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } @Override - public ImprovedKTable reduce(final Reducer reducer, final Named named, + public KTableX reduce(final Reducer reducer, final Named named, final AutoMaterialized> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator aggregator) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator aggregator, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator aggregator, final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, named, materialized)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedTimeWindowedKStream windowedBy(final Windows windows) { + public TimeWindowedKStreamX windowedBy(final Windows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override - public ImprovedTimeWindowedKStream windowedBy(final SlidingWindows windows) { + public TimeWindowedKStreamX windowedBy(final SlidingWindows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); } @Override - public ImprovedSessionWindowedKStream windowedBy(final SessionWindows windows) { + public SessionWindowedKStreamX windowedBy(final SessionWindows windows) { return this.context.wrap(this.wrapped.windowedBy(windows)); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java similarity index 68% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java index 5674c201..9773160c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java @@ -39,83 +39,83 @@ * @param type of keys * @param type of values */ -public interface ImprovedKGroupedTable extends KGroupedTable { +public interface KGroupedTableX extends KGroupedTable { @Override - ImprovedKTable count(Materialized> materialized); + KTableX count(Materialized> materialized); /** * @see #count(Materialized) */ - ImprovedKTable count(AutoMaterialized> materialized); + KTableX count(AutoMaterialized> materialized); @Override - ImprovedKTable count(Named named, Materialized> materialized); + KTableX count(Named named, Materialized> materialized); /** * @see #count(Named, Materialized) */ - ImprovedKTable count(Named named, + KTableX count(Named named, AutoMaterialized> materialized); @Override - ImprovedKTable count(); + KTableX count(); @Override - ImprovedKTable count(Named named); + KTableX count(Named named); @Override - ImprovedKTable reduce(Reducer adder, Reducer subtractor, + KTableX reduce(Reducer adder, Reducer subtractor, Materialized> materialized); /** * @see #reduce(Reducer, Reducer, Materialized) */ - ImprovedKTable reduce(Reducer adder, Reducer subtractor, + KTableX reduce(Reducer adder, Reducer subtractor, AutoMaterialized> materialized); @Override - ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, + KTableX reduce(Reducer adder, Reducer subtractor, Named named, Materialized> materialized); /** * @see #reduce(Reducer, Reducer, Named, Materialized) */ - ImprovedKTable reduce(Reducer adder, Reducer subtractor, Named named, + KTableX reduce(Reducer adder, Reducer subtractor, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable reduce(Reducer adder, Reducer subtractor); + KTableX reduce(Reducer adder, Reducer subtractor); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Aggregator, Materialized) */ - ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, AutoMaterialized> materialized); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Aggregator, Named, Materialized) */ - ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor); @Override - ImprovedKTable aggregate(Initializer initializer, Aggregator adder, + KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java similarity index 78% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java index a6304291..ab40e603 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKGroupedTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java @@ -36,75 +36,75 @@ import org.apache.kafka.streams.state.KeyValueStore; @RequiredArgsConstructor -class ImprovedKGroupedTableImpl implements ImprovedKGroupedTable { +class KGroupedTableXImpl implements KGroupedTableX { private final @NonNull KGroupedTable wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable count(final Materialized> materialized) { + public KTableX count(final Materialized> materialized) { return this.context.wrap(this.wrapped.count(materialized)); } @Override - public ImprovedKTable count( + public KTableX count( final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable count(final Named named, + public KTableX count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } @Override - public ImprovedKTable count(final Named named, + public KTableX count(final Named named, final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable count() { + public KTableX count() { return this.context.wrap(this.wrapped.count()); } @Override - public ImprovedKTable count(final Named named) { + public KTableX count(final Named named) { return this.context.wrap(this.wrapped.count(named)); } @Override - public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, + public KTableX reduce(final Reducer adder, final Reducer subtractor, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(adder, subtractor, materialized)); } @Override - public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, + public KTableX reduce(final Reducer adder, final Reducer subtractor, final AutoMaterialized> materialized) { return this.reduce(adder, subtractor, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, + public KTableX reduce(final Reducer adder, final Reducer subtractor, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(adder, subtractor, materialized)); } @Override - public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor, final Named named, + public KTableX reduce(final Reducer adder, final Reducer subtractor, final Named named, final AutoMaterialized> materialized) { return this.reduce(adder, subtractor, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable reduce(final Reducer adder, final Reducer subtractor) { + public KTableX reduce(final Reducer adder, final Reducer subtractor) { return this.context.wrap(this.wrapped.reduce(adder, subtractor)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Materialized> materialized) { @@ -112,7 +112,7 @@ public ImprovedKTable aggregate(final Initializer initializer, } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final AutoMaterialized> materialized) { @@ -120,7 +120,7 @@ public ImprovedKTable aggregate(final Initializer initializer, } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named, final Materialized> materialized) { @@ -128,7 +128,7 @@ public ImprovedKTable aggregate(final Initializer initializer, } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named, final AutoMaterialized> materialized) { @@ -137,14 +137,14 @@ public ImprovedKTable aggregate(final Initializer initializer, } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor) { return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor)); } @Override - public ImprovedKTable aggregate(final Initializer initializer, + public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named) { return this.context.wrap(this.wrapped.aggregate(initializer, adder, subtractor, named)); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java similarity index 64% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index d309eca4..af6773cd 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTable.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -47,110 +47,110 @@ * @param type of keys * @param type of values */ -public interface ImprovedKTable extends KTable { +public interface KTableX extends KTable { @Override - ImprovedKTable filter(Predicate predicate); + KTableX filter(Predicate predicate); @Override - ImprovedKTable filter(Predicate predicate, Named named); + KTableX filter(Predicate predicate, Named named); @Override - ImprovedKTable filter(Predicate predicate, + KTableX filter(Predicate predicate, Materialized> materialized); /** * @see #filter(Predicate, Materialized) */ - ImprovedKTable filter(Predicate predicate, + KTableX filter(Predicate predicate, AutoMaterialized> materialized); @Override - ImprovedKTable filter(Predicate predicate, Named named, + KTableX filter(Predicate predicate, Named named, Materialized> materialized); /** * @see #filter(Predicate, Named, Materialized) */ - ImprovedKTable filter(Predicate predicate, Named named, + KTableX filter(Predicate predicate, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable filterNot(Predicate predicate); + KTableX filterNot(Predicate predicate); @Override - ImprovedKTable filterNot(Predicate predicate, Named named); + KTableX filterNot(Predicate predicate, Named named); @Override - ImprovedKTable filterNot(Predicate predicate, + KTableX filterNot(Predicate predicate, Materialized> materialized); /** * @see #filterNot(Predicate, Materialized) */ - ImprovedKTable filterNot(Predicate predicate, + KTableX filterNot(Predicate predicate, AutoMaterialized> materialized); @Override - ImprovedKTable filterNot(Predicate predicate, Named named, + KTableX filterNot(Predicate predicate, Named named, Materialized> materialized); /** * @see #filterNot(Predicate, Named, Materialized) */ - ImprovedKTable filterNot(Predicate predicate, Named named, + KTableX filterNot(Predicate predicate, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable mapValues(ValueMapper mapper); + KTableX mapValues(ValueMapper mapper); @Override - ImprovedKTable mapValues(ValueMapper mapper, Named named); + KTableX mapValues(ValueMapper mapper, Named named); @Override - ImprovedKTable mapValues(ValueMapperWithKey mapper); + KTableX mapValues(ValueMapperWithKey mapper); @Override - ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named); + KTableX mapValues(ValueMapperWithKey mapper, Named named); @Override - ImprovedKTable mapValues(ValueMapper mapper, + KTableX mapValues(ValueMapper mapper, Materialized> materialized); /** * @see #mapValues(ValueMapper, Materialized) */ - ImprovedKTable mapValues(ValueMapper mapper, + KTableX mapValues(ValueMapper mapper, AutoMaterialized> materialized); @Override - ImprovedKTable mapValues(ValueMapper mapper, Named named, + KTableX mapValues(ValueMapper mapper, Named named, Materialized> materialized); /** * @see #mapValues(ValueMapper, Named, Materialized) */ - ImprovedKTable mapValues(ValueMapper mapper, Named named, + KTableX mapValues(ValueMapper mapper, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable mapValues(ValueMapperWithKey mapper, + KTableX mapValues(ValueMapperWithKey mapper, Materialized> materialized); /** * @see #mapValues(ValueMapperWithKey, Materialized) */ - ImprovedKTable mapValues(ValueMapperWithKey mapper, + KTableX mapValues(ValueMapperWithKey mapper, AutoMaterialized> materialized); @Override - ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, + KTableX mapValues(ValueMapperWithKey mapper, Named named, Materialized> materialized); /** * @see #mapValues(ValueMapperWithKey, Named, Materialized) */ - ImprovedKTable mapValues(ValueMapperWithKey mapper, Named named, + KTableX mapValues(ValueMapperWithKey mapper, Named named, AutoMaterialized> materialized); @Override @@ -166,236 +166,236 @@ ImprovedKTable mapValues(ValueMapperWithKey ImprovedKStream toStream(KeyValueMapper mapper, Named named); @Override - ImprovedKTable suppress(Suppressed suppressed); + KTableX suppress(Suppressed suppressed); @Override - ImprovedKTable transformValues( + KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, String... stateStoreNames); @Override - ImprovedKTable transformValues( + KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKTable transformValues( + KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, Materialized> materialized, String... stateStoreNames); /** * @see #transformValues(ValueTransformerWithKeySupplier, Materialized, String...) */ - ImprovedKTable transformValues( + KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, AutoMaterialized> materialized, String... stateStoreNames); @Override - ImprovedKTable transformValues( + KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, Materialized> materialized, Named named, String... stateStoreNames); /** * @see #transformValues(ValueTransformerWithKeySupplier, Materialized, Named, String...) */ - ImprovedKTable transformValues( + KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, AutoMaterialized> materialized, Named named, String... stateStoreNames); @Override - ImprovedKGroupedTable groupBy(KeyValueMapper> selector); + KGroupedTableX groupBy(KeyValueMapper> selector); @Override - ImprovedKGroupedTable groupBy(KeyValueMapper> selector, + KGroupedTableX groupBy(KeyValueMapper> selector, Grouped grouped); /** * @see #groupBy(KeyValueMapper, Grouped) */ - ImprovedKGroupedTable groupBy(KeyValueMapper> selector, + KGroupedTableX groupBy(KeyValueMapper> selector, AutoGrouped grouped); @Override - ImprovedKTable join(KTable other, ValueJoiner joiner); + KTableX join(KTable other, ValueJoiner joiner); @Override - ImprovedKTable join(KTable other, ValueJoiner joiner, + KTableX join(KTable other, ValueJoiner joiner, Named named); @Override - ImprovedKTable join(KTable other, ValueJoiner joiner, + KTableX join(KTable other, ValueJoiner joiner, Materialized> materialized); /** * @see #join(KTable, ValueJoiner, Materialized) */ - ImprovedKTable join(KTable other, ValueJoiner joiner, + KTableX join(KTable other, ValueJoiner joiner, AutoMaterialized> materialized); @Override - ImprovedKTable join(KTable other, ValueJoiner joiner, + KTableX join(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); /** * @see #join(KTable, ValueJoiner, Named, Materialized) */ - ImprovedKTable join(KTable other, ValueJoiner joiner, + KTableX join(KTable other, ValueJoiner joiner, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable leftJoin(KTable other, + KTableX leftJoin(KTable other, ValueJoiner joiner); @Override - ImprovedKTable leftJoin(KTable other, + KTableX leftJoin(KTable other, ValueJoiner joiner, Named named); @Override - ImprovedKTable leftJoin(KTable other, + KTableX leftJoin(KTable other, ValueJoiner joiner, Materialized> materialized); /** * @see #leftJoin(KTable, ValueJoiner, Materialized) */ - ImprovedKTable leftJoin(KTable other, + KTableX leftJoin(KTable other, ValueJoiner joiner, AutoMaterialized> materialized); @Override - ImprovedKTable leftJoin(KTable other, + KTableX leftJoin(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); /** * @see #leftJoin(KTable, ValueJoiner, Named, Materialized) */ - ImprovedKTable leftJoin(KTable other, + KTableX leftJoin(KTable other, ValueJoiner joiner, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable outerJoin(KTable other, + KTableX outerJoin(KTable other, ValueJoiner joiner); @Override - ImprovedKTable outerJoin(KTable other, + KTableX outerJoin(KTable other, ValueJoiner joiner, Named named); @Override - ImprovedKTable outerJoin(KTable other, + KTableX outerJoin(KTable other, ValueJoiner joiner, Materialized> materialized); /** * @see #outerJoin(KTable, ValueJoiner, Materialized) */ - ImprovedKTable outerJoin(KTable other, + KTableX outerJoin(KTable other, ValueJoiner joiner, AutoMaterialized> materialized); @Override - ImprovedKTable outerJoin(KTable other, + KTableX outerJoin(KTable other, ValueJoiner joiner, Named named, Materialized> materialized); /** * @see #outerJoin(KTable, ValueJoiner, Named, Materialized) */ - ImprovedKTable outerJoin(KTable other, + KTableX outerJoin(KTable other, ValueJoiner joiner, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); @Override - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named); @Override - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined); @Override - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Materialized> materialized); /** * @see #join(KTable, Function, ValueJoiner, Materialized) */ - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, AutoMaterialized> materialized); @Override - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); /** * @see #join(KTable, Function, ValueJoiner, Named, Materialized) */ - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, Materialized> materialized); /** * @see #join(KTable, Function, ValueJoiner, TableJoined, Materialized) */ - ImprovedKTable join(KTable other, Function foreignKeyExtractor, + KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, AutoMaterialized> materialized); @Override - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); @Override - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named); @Override - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined); @Override - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Materialized> materialized); /** * @see #leftJoin(KTable, Function, ValueJoiner, Materialized) */ - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, AutoMaterialized> materialized); @Override - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); /** * @see #leftJoin(KTable, Function, ValueJoiner, Named, Materialized) */ - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, Materialized> materialized); /** * @see #leftJoin(KTable, Function, ValueJoiner, TableJoined, Materialized) */ - ImprovedKTable leftJoin(KTable other, Function foreignKeyExtractor, + KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, AutoMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java similarity index 78% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java index 2cdac686..2a773888 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKTableImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java @@ -46,145 +46,145 @@ import org.apache.kafka.streams.state.KeyValueStore; @RequiredArgsConstructor -class ImprovedKTableImpl implements ImprovedKTable { +class KTableXImpl implements KTableX { @Getter(AccessLevel.PROTECTED) private final @NonNull KTable wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable filter(final Predicate predicate) { + public KTableX filter(final Predicate predicate) { return this.context.wrap(this.wrapped.filter(predicate)); } @Override - public ImprovedKTable filter(final Predicate predicate, final Named named) { + public KTableX filter(final Predicate predicate, final Named named) { return this.context.wrap(this.wrapped.filter(predicate, named)); } @Override - public ImprovedKTable filter(final Predicate predicate, + public KTableX filter(final Predicate predicate, final Materialized> materialized) { return this.context.wrap(this.wrapped.filter(predicate, materialized)); } @Override - public ImprovedKTable filter(final Predicate predicate, + public KTableX filter(final Predicate predicate, final AutoMaterialized> materialized) { return this.filter(predicate, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable filter(final Predicate predicate, final Named named, + public KTableX filter(final Predicate predicate, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.filter(predicate, named, materialized)); } @Override - public ImprovedKTable filter(final Predicate predicate, final Named named, + public KTableX filter(final Predicate predicate, final Named named, final AutoMaterialized> materialized) { return this.filter(predicate, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable filterNot(final Predicate predicate) { + public KTableX filterNot(final Predicate predicate) { return this.context.wrap(this.wrapped.filterNot(predicate)); } @Override - public ImprovedKTable filterNot(final Predicate predicate, final Named named) { + public KTableX filterNot(final Predicate predicate, final Named named) { return this.context.wrap(this.wrapped.filterNot(predicate, named)); } @Override - public ImprovedKTable filterNot(final Predicate predicate, + public KTableX filterNot(final Predicate predicate, final Materialized> materialized) { return this.context.wrap(this.wrapped.filterNot(predicate, materialized)); } @Override - public ImprovedKTable filterNot(final Predicate predicate, + public KTableX filterNot(final Predicate predicate, final AutoMaterialized> materialized) { return this.filterNot(predicate, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable filterNot(final Predicate predicate, final Named named, + public KTableX filterNot(final Predicate predicate, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.filterNot(predicate, materialized)); } @Override - public ImprovedKTable filterNot(final Predicate predicate, final Named named, + public KTableX filterNot(final Predicate predicate, final Named named, final AutoMaterialized> materialized) { return this.filterNot(predicate, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable mapValues(final ValueMapper mapper) { + public KTableX mapValues(final ValueMapper mapper) { return this.context.wrap(this.wrapped.mapValues(mapper)); } @Override - public ImprovedKTable mapValues(final ValueMapper mapper, final Named named) { + public KTableX mapValues(final ValueMapper mapper, final Named named) { return this.context.wrap(this.wrapped.mapValues(mapper, named)); } @Override - public ImprovedKTable mapValues(final ValueMapperWithKey mapper) { + public KTableX mapValues(final ValueMapperWithKey mapper) { return this.context.wrap(this.wrapped.mapValues(mapper)); } @Override - public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + public KTableX mapValues(final ValueMapperWithKey mapper, final Named named) { return this.context.wrap(this.wrapped.mapValues(mapper, named)); } @Override - public ImprovedKTable mapValues(final ValueMapper mapper, + public KTableX mapValues(final ValueMapper mapper, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, materialized)); } @Override - public ImprovedKTable mapValues(final ValueMapper mapper, + public KTableX mapValues(final ValueMapper mapper, final AutoMaterialized> materialized) { return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable mapValues(final ValueMapper mapper, final Named named, + public KTableX mapValues(final ValueMapper mapper, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, named, materialized)); } @Override - public ImprovedKTable mapValues(final ValueMapper mapper, final Named named, + public KTableX mapValues(final ValueMapper mapper, final Named named, final AutoMaterialized> materialized) { return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + public KTableX mapValues(final ValueMapperWithKey mapper, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, materialized)); } @Override - public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + public KTableX mapValues(final ValueMapperWithKey mapper, final AutoMaterialized> materialized) { return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + public KTableX mapValues(final ValueMapperWithKey mapper, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.mapValues(mapper, named, materialized)); } @Override - public ImprovedKTable mapValues(final ValueMapperWithKey mapper, + public KTableX mapValues(final ValueMapperWithKey mapper, final Named named, final AutoMaterialized> materialized) { return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); } @@ -211,19 +211,19 @@ public ImprovedKStream toStream(final KeyValueMapper suppress(final Suppressed suppressed) { + public KTableX suppress(final Suppressed suppressed) { return this.context.wrap(this.wrapped.suppress(suppressed)); } @Override - public ImprovedKTable transformValues( + public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transformValues(transformerSupplier, stateStoreNames)); } @Override - public ImprovedKTable transformValues( + public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final Named named, final String... stateStoreNames) { @@ -231,14 +231,14 @@ public ImprovedKTable transformValues( } @Override - public ImprovedKTable transformValues( + public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final Materialized> materialized, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transformValues(transformerSupplier, materialized, stateStoreNames)); } @Override - public ImprovedKTable transformValues( + public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final AutoMaterialized> materialized, final String... stateStoreNames) { @@ -247,7 +247,7 @@ public ImprovedKTable transformValues( } @Override - public ImprovedKTable transformValues( + public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final Materialized> materialized, final Named named, final String... stateStoreNames) { @@ -256,7 +256,7 @@ public ImprovedKTable transformValues( } @Override - public ImprovedKTable transformValues( + public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, final AutoMaterialized> materialized, final Named named, final String... stateStoreNames) { @@ -265,39 +265,39 @@ public ImprovedKTable transformValues( } @Override - public ImprovedKGroupedTable groupBy( + public KGroupedTableX groupBy( final KeyValueMapper> selector) { return this.context.wrap(this.wrapped.groupBy(selector)); } @Override - public ImprovedKGroupedTable groupBy( + public KGroupedTableX groupBy( final KeyValueMapper> selector, final Grouped grouped) { return this.context.wrap(this.wrapped.groupBy(selector, grouped)); } @Override - public ImprovedKGroupedTable groupBy( + public KGroupedTableX groupBy( final KeyValueMapper> selector, final AutoGrouped grouped) { return this.groupBy(selector, grouped.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final ValueJoiner joiner) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap(this.wrapped.join(otherTable, joiner)); } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final ValueJoiner joiner, final Named named) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap(this.wrapped.join(otherTable, joiner, named)); } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -305,14 +305,14 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final ValueJoiner joiner, final AutoMaterialized> materialized) { return this.join(other, joiner, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -320,28 +320,28 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final ValueJoiner joiner, final Named named, final AutoMaterialized> materialized) { return this.join(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final ValueJoiner joiner) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner)); } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final ValueJoiner joiner, final Named named) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap(this.wrapped.leftJoin(otherTable, joiner, named)); } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -349,14 +349,14 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final ValueJoiner joiner, final AutoMaterialized> materialized) { return this.leftJoin(other, joiner, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -364,28 +364,28 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final ValueJoiner joiner, final Named named, final AutoMaterialized> materialized) { return this.leftJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable outerJoin(final KTable other, + public KTableX outerJoin(final KTable other, final ValueJoiner joiner) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner)); } @Override - public ImprovedKTable outerJoin(final KTable other, + public KTableX outerJoin(final KTable other, final ValueJoiner joiner, final Named named) { final KTable otherTable = StreamsContext.maybeUnwrap(other); return this.context.wrap(this.wrapped.outerJoin(otherTable, joiner, named)); } @Override - public ImprovedKTable outerJoin(final KTable other, + public KTableX outerJoin(final KTable other, final ValueJoiner joiner, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -393,14 +393,14 @@ public ImprovedKTable outerJoin(final KTable other, } @Override - public ImprovedKTable outerJoin(final KTable other, + public KTableX outerJoin(final KTable other, final ValueJoiner joiner, final AutoMaterialized> materialized) { return this.outerJoin(other, joiner, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable outerJoin(final KTable other, + public KTableX outerJoin(final KTable other, final ValueJoiner joiner, final Named named, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -408,14 +408,14 @@ public ImprovedKTable outerJoin(final KTable other, } @Override - public ImprovedKTable outerJoin(final KTable other, + public KTableX outerJoin(final KTable other, final ValueJoiner joiner, final Named named, final AutoMaterialized> materialized) { return this.outerJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -423,7 +423,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -431,7 +431,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -439,7 +439,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -447,7 +447,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final AutoMaterialized> materialized) { @@ -455,7 +455,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final Materialized> materialized) { @@ -464,7 +464,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final AutoMaterialized> materialized) { @@ -473,7 +473,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { @@ -482,7 +482,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable join(final KTable other, + public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final AutoMaterialized> materialized) { @@ -491,7 +491,7 @@ public ImprovedKTable join(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -499,7 +499,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -507,7 +507,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -515,7 +515,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Materialized> materialized) { final KTable otherTable = StreamsContext.maybeUnwrap(other); @@ -523,7 +523,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final AutoMaterialized> materialized) { @@ -532,7 +532,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final Materialized> materialized) { @@ -541,7 +541,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, final AutoMaterialized> materialized) { @@ -550,7 +550,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final Materialized> materialized) { @@ -560,7 +560,7 @@ public ImprovedKTable leftJoin(final KTable other, } @Override - public ImprovedKTable leftJoin(final KTable other, + public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, final AutoMaterialized> materialized) { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java similarity index 74% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java index 9c02b827..2b0b95aa 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java @@ -39,32 +39,32 @@ * @param type of keys * @param type of values */ -public interface ImprovedSessionWindowedCogroupedKStream extends SessionWindowedCogroupedKStream { +public interface SessionWindowedCogroupedKStreamX extends SessionWindowedCogroupedKStream { @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger); + KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger); @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger, Named named); @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger, Materialized> materialized); /** * @see #aggregate(Initializer, Merger, Materialized) */ - ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger, AutoMaterialized> materialized); @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger, Named named, Materialized> materialized); /** * @see #aggregate(Initializer, Merger, Named, Materialized) */ - ImprovedKTable, VOut> aggregate(Initializer initializer, Merger sessionMerger, + KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger, Named named, AutoMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java similarity index 83% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java index e5e19819..9bd8baff 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java @@ -36,47 +36,47 @@ import org.apache.kafka.streams.state.SessionStore; @RequiredArgsConstructor -class ImprovedSessionWindowedCogroupedStreamImpl implements ImprovedSessionWindowedCogroupedKStream { +class SessionWindowedCogroupedStreamXImpl implements SessionWindowedCogroupedKStreamX { private final @NonNull SessionWindowedCogroupedKStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger) { return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Named named) { return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, named)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, materialized)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, final AutoMaterialized> materialized) { return this.aggregate(initializer, sessionMerger, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, sessionMerger, materialized)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, final Named named, final AutoMaterialized> materialized) { return this.aggregate(initializer, sessionMerger, named, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java similarity index 72% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java index 935a3aa5..616ba980 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java @@ -42,57 +42,57 @@ * @param type of keys * @param type of values */ -public interface ImprovedSessionWindowedKStream extends SessionWindowedKStream { +public interface SessionWindowedKStreamX extends SessionWindowedKStream { @Override - ImprovedKTable, Long> count(); + KTableX, Long> count(); @Override - ImprovedKTable, Long> count(Named named); + KTableX, Long> count(Named named); @Override - ImprovedKTable, Long> count(Materialized> materialized); + KTableX, Long> count(Materialized> materialized); /** * @see #count(Materialized) */ - ImprovedKTable, Long> count(AutoMaterialized> materialized); + KTableX, Long> count(AutoMaterialized> materialized); @Override - ImprovedKTable, Long> count(Named named, + KTableX, Long> count(Named named, Materialized> materialized); /** * @see #count(Named, Materialized) */ - ImprovedKTable, Long> count(Named named, + KTableX, Long> count(Named named, AutoMaterialized> materialized); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Merger, Materialized) */ - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, AutoMaterialized> materialized); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named, Materialized> materialized); @@ -100,37 +100,37 @@ ImprovedKTable, VR> aggregate(Initializer initializer, /** * @see #aggregate(Initializer, Aggregator, Merger, Named, Materialized) */ - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable, V> reduce(Reducer reducer); + KTableX, V> reduce(Reducer reducer); @Override - ImprovedKTable, V> reduce(Reducer reducer, Named named); + KTableX, V> reduce(Reducer reducer, Named named); @Override - ImprovedKTable, V> reduce(Reducer reducer, + KTableX, V> reduce(Reducer reducer, Materialized> materialized); /** * @see #reduce(Reducer, Materialized) */ - ImprovedKTable, V> reduce(Reducer reducer, + KTableX, V> reduce(Reducer reducer, AutoMaterialized> materialized); @Override - ImprovedKTable, V> reduce(Reducer reducer, Named named, + KTableX, V> reduce(Reducer reducer, Named named, Materialized> materialized); /** * @see #reduce(Reducer, Named, Materialized) */ - ImprovedKTable, V> reduce(Reducer reducer, Named named, + KTableX, V> reduce(Reducer reducer, Named named, AutoMaterialized> materialized); @Override - ImprovedSessionWindowedKStream emitStrategy(EmitStrategy emitStrategy); + SessionWindowedKStreamX emitStrategy(EmitStrategy emitStrategy); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java similarity index 78% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java index ecfe2646..1b21a728 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedSessionWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java @@ -39,67 +39,67 @@ import org.apache.kafka.streams.state.SessionStore; @RequiredArgsConstructor -class ImprovedSessionWindowedStreamImpl implements ImprovedSessionWindowedKStream { +class SessionWindowedStreamXImpl implements SessionWindowedKStreamX { private final @NonNull SessionWindowedKStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable, Long> count() { + public KTableX, Long> count() { return this.context.wrap(this.wrapped.count()); } @Override - public ImprovedKTable, Long> count(final Named named) { + public KTableX, Long> count(final Named named) { return this.context.wrap(this.wrapped.count(named)); } @Override - public ImprovedKTable, Long> count( + public KTableX, Long> count( final Materialized> materialized) { return this.context.wrap(this.wrapped.count(materialized)); } @Override - public ImprovedKTable, Long> count( + public KTableX, Long> count( final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, Long> count(final Named named, + public KTableX, Long> count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } @Override - public ImprovedKTable, Long> count(final Named named, + public KTableX, Long> count(final Named named, final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Named named) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, named)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, sessionMerger, materialized)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, sessionMerger, @@ -107,7 +107,7 @@ public ImprovedKTable, VR> aggregate(final Initializer init } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Named named, final Materialized> materialized) { @@ -115,7 +115,7 @@ public ImprovedKTable, VR> aggregate(final Initializer init } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Named named, final AutoMaterialized> materialized) { @@ -124,41 +124,41 @@ public ImprovedKTable, VR> aggregate(final Initializer init } @Override - public ImprovedKTable, V> reduce(final Reducer reducer) { + public KTableX, V> reduce(final Reducer reducer) { return this.context.wrap(this.wrapped.reduce(reducer)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, final Named named) { + public KTableX, V> reduce(final Reducer reducer, final Named named) { return this.context.wrap(this.wrapped.reduce(reducer, named)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, + public KTableX, V> reduce(final Reducer reducer, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, + public KTableX, V> reduce(final Reducer reducer, final AutoMaterialized> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + public KTableX, V> reduce(final Reducer reducer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + public KTableX, V> reduce(final Reducer reducer, final Named named, final AutoMaterialized> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedSessionWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { + public SessionWindowedKStreamX emitStrategy(final EmitStrategy emitStrategy) { return this.context.wrap(this.wrapped.emitStrategy(emitStrategy)); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index 9417de42..73382fba 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -61,25 +61,25 @@ static KStream maybeUnwrap(final KStream stream) { } static KGroupedStream maybeUnwrap(final KGroupedStream stream) { - if (stream instanceof ImprovedKGroupedStream) { + if (stream instanceof KGroupedStreamX) { // Kafka Streams internally casts KGroupedStream to KGroupedStreamImpl in some cases - return ((ImprovedKGroupedStreamImpl) stream).getWrapped(); + return ((KGroupedStreamXImpl) stream).getWrapped(); } return stream; } static KTable maybeUnwrap(final KTable table) { - if (table instanceof ImprovedKTableImpl) { + if (table instanceof KTableXImpl) { // Kafka Streams internally casts KTable to KTableImpl in some cases - return ((ImprovedKTableImpl) table).getWrapped(); + return ((KTableXImpl) table).getWrapped(); } return table; } /** - * Wrap a {@code KStream} and add methods to simplify Serde configuration, error handling, and topic access + * Wrap a {@link KStream} and add methods to simplify Serde configuration, error handling, and topic access * @param stream stream to be wrapped - * @return {@code ImprovedKStream} + * @return {@link ImprovedKStream} * @param type of keys in the stream * @param type of values in the stream */ @@ -91,87 +91,87 @@ public ImprovedKStream wrap(final KStream stream) { } /** - * Wrap a {@code KGroupedStream} and add methods to simplify Serde configuration, error handling, and topic access + * Wrap a {@link KGroupedStream} and add methods to simplify Serde configuration, error handling, and topic access * @param stream stream to be wrapped - * @return {@code ImprovedKGroupedStream} + * @return {@link KGroupedStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedKGroupedStream wrap(final KGroupedStream stream) { - if (stream instanceof ImprovedKGroupedStream) { - return (ImprovedKGroupedStream) stream; + public KGroupedStreamX wrap(final KGroupedStream stream) { + if (stream instanceof KGroupedStreamX) { + return (KGroupedStreamX) stream; } - return new ImprovedKGroupedStreamImpl<>(stream, this); + return new KGroupedStreamXImpl<>(stream, this); } /** - * Wrap a {@code TimeWindowedKStream} and add methods to simplify Serde configuration, error handling, and topic + * Wrap a {@link TimeWindowedKStream} and add methods to simplify Serde configuration, error handling, and topic * access * @param stream stream to be wrapped - * @return {@code ImprovedTimeWindowedKStream} + * @return {@link TimeWindowedKStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedTimeWindowedKStream wrap( + public TimeWindowedKStreamX wrap( final TimeWindowedKStream stream) { - if (stream instanceof ImprovedTimeWindowedKStream) { - return (ImprovedTimeWindowedKStream) stream; + if (stream instanceof TimeWindowedKStreamX) { + return (TimeWindowedKStreamX) stream; } - return new ImprovedTimeWindowedStreamImpl<>(stream, this); + return new TimeWindowedStreamXImpl<>(stream, this); } /** - * Wrap a {@code SessionWindowedKStream} and add methods to simplify Serde configuration, error handling, and + * Wrap a {@link SessionWindowedKStream} and add methods to simplify Serde configuration, error handling, and * topic access * @param stream stream to be wrapped - * @return {@code ImprovedSessionWindowedKStream} + * @return {@link SessionWindowedKStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedSessionWindowedKStream wrap( + public SessionWindowedKStreamX wrap( final SessionWindowedKStream stream) { - if (stream instanceof ImprovedSessionWindowedKStream) { - return (ImprovedSessionWindowedKStream) stream; + if (stream instanceof SessionWindowedKStreamX) { + return (SessionWindowedKStreamX) stream; } - return new ImprovedSessionWindowedStreamImpl<>(stream, this); + return new SessionWindowedStreamXImpl<>(stream, this); } /** - * Wrap a {@code TimeWindowedCogroupedKStream} and add methods to simplify Serde configuration, error handling, + * Wrap a {@link TimeWindowedCogroupedKStream} and add methods to simplify Serde configuration, error handling, * and topic access * @param stream stream to be wrapped - * @return {@code ImprovedTimeWindowedCogroupedKStream} + * @return {@link TimeWindowedCogroupedKStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedTimeWindowedCogroupedKStream wrap( + public TimeWindowedCogroupedKStreamX wrap( final TimeWindowedCogroupedKStream stream) { - if (stream instanceof ImprovedTimeWindowedCogroupedKStream) { - return (ImprovedTimeWindowedCogroupedKStream) stream; + if (stream instanceof TimeWindowedCogroupedKStreamX) { + return (TimeWindowedCogroupedKStreamX) stream; } - return new ImprovedTimeWindowedCogroupedStreamImpl<>(stream, this); + return new TimeWindowedCogroupedStreamXImpl<>(stream, this); } /** - * Wrap a {@code SessionWindowedCogroupedKStream} and add methods to simplify Serde configuration, error + * Wrap a {@link SessionWindowedCogroupedKStream} and add methods to simplify Serde configuration, error * handling, and topic access * @param stream stream to be wrapped - * @return {@code ImprovedSessionWindowedCogroupedKStream} + * @return {@link SessionWindowedCogroupedKStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedSessionWindowedCogroupedKStream wrap( + public SessionWindowedCogroupedKStreamX wrap( final SessionWindowedCogroupedKStream stream) { - if (stream instanceof ImprovedSessionWindowedCogroupedKStream) { - return (ImprovedSessionWindowedCogroupedKStream) stream; + if (stream instanceof SessionWindowedCogroupedKStreamX) { + return (SessionWindowedCogroupedKStreamX) stream; } - return new ImprovedSessionWindowedCogroupedStreamImpl<>(stream, this); + return new SessionWindowedCogroupedStreamXImpl<>(stream, this); } /** - * Wrap a {@code CogroupedKStream} and add methods to simplify Serde configuration, error handling, and topic access + * Wrap a {@link CogroupedKStream} and add methods to simplify Serde configuration, error handling, and topic access * @param stream stream to be wrapped - * @return {@code ImprovedCogroupedKStream} + * @return {@link CogroupedKStreamX} * @param type of keys in the stream * @param type of values in the stream */ @@ -183,44 +183,44 @@ public CogroupedKStreamX wrap(final CogroupedKStream st } /** - * Wrap a {@code BranchedKStream} and add methods to simplify Serde configuration, error handling, and topic access + * Wrap a {@link BranchedKStream} and add methods to simplify Serde configuration, error handling, and topic access * @param stream stream to be wrapped - * @return {@code ImprovedBranchedKStream} + * @return {@link BranchedKStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedBranchedKStream wrap(final BranchedKStream stream) { - if (stream instanceof ImprovedBranchedKStream) { - return (ImprovedBranchedKStream) stream; + public BranchedKStreamX wrap(final BranchedKStream stream) { + if (stream instanceof BranchedKStreamX) { + return (BranchedKStreamX) stream; } - return new ImprovedBranchedKStreamImpl<>(stream, this); + return new BranchedKStreamXImpl<>(stream, this); } /** - * Wrap a {@code KTable} and add methods to simplify Serde configuration, error handling, and topic access + * Wrap a {@link KTable} and add methods to simplify Serde configuration, error handling, and topic access * @param table table to be wrapped - * @return {@code ImprovedKTable} + * @return {@link KTableX} * @param type of keys in the table * @param type of values in the table */ - public ImprovedKTable wrap(final KTable table) { - if (table instanceof ImprovedKTable) { - return (ImprovedKTable) table; + public KTableX wrap(final KTable table) { + if (table instanceof KTableX) { + return (KTableX) table; } - return new ImprovedKTableImpl<>(table, this); + return new KTableXImpl<>(table, this); } /** - * Wrap a {@code KGroupedTable} and add methods to simplify Serde configuration, error handling, and topic access + * Wrap a {@link KGroupedTable} and add methods to simplify Serde configuration, error handling, and topic access * @param table table to be wrapped - * @return {@code ImprovedKGroupedTable} + * @return {@link KGroupedTableX} * @param type of keys in the table * @param type of values in the table */ - public ImprovedKGroupedTable wrap(final KGroupedTable table) { - if (table instanceof ImprovedKGroupedTable) { - return (ImprovedKGroupedTable) table; + public KGroupedTableX wrap(final KGroupedTable table) { + if (table instanceof KGroupedTableX) { + return (KGroupedTableX) table; } - return new ImprovedKGroupedTableImpl<>(table, this); + return new KGroupedTableXImpl<>(table, this); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java similarity index 78% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java index 7f21d401..92bbc6b5 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java @@ -38,31 +38,31 @@ * @param type of keys * @param type of values */ -public interface ImprovedTimeWindowedCogroupedKStream extends TimeWindowedCogroupedKStream { +public interface TimeWindowedCogroupedKStreamX extends TimeWindowedCogroupedKStream { @Override - ImprovedKTable, VOut> aggregate(Initializer initializer); + KTableX, VOut> aggregate(Initializer initializer); @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, Named named); + KTableX, VOut> aggregate(Initializer initializer, Named named); @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, + KTableX, VOut> aggregate(Initializer initializer, Materialized> materialized); /** * @see #aggregate(Initializer, Materialized) */ - ImprovedKTable, VOut> aggregate(Initializer initializer, + KTableX, VOut> aggregate(Initializer initializer, AutoMaterialized> materialized); @Override - ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, + KTableX, VOut> aggregate(Initializer initializer, Named named, Materialized> materialized); /** * @see #aggregate(Initializer, Named, Materialized) */ - ImprovedKTable, VOut> aggregate(Initializer initializer, Named named, + KTableX, VOut> aggregate(Initializer initializer, Named named, AutoMaterialized> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java similarity index 79% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java index 63982d89..f4d96319 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedCogroupedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java @@ -35,41 +35,41 @@ import org.apache.kafka.streams.state.WindowStore; @RequiredArgsConstructor -class ImprovedTimeWindowedCogroupedStreamImpl implements ImprovedTimeWindowedCogroupedKStream { +class TimeWindowedCogroupedStreamXImpl implements TimeWindowedCogroupedKStreamX { private final @NonNull TimeWindowedCogroupedKStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable, V> aggregate(final Initializer initializer) { + public KTableX, V> aggregate(final Initializer initializer) { return this.context.wrap(this.wrapped.aggregate(initializer)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named) { + public KTableX, V> aggregate(final Initializer initializer, final Named named) { return this.context.wrap(this.wrapped.aggregate(initializer, named)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, materialized)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, + public KTableX, V> aggregate(final Initializer initializer, final AutoMaterialized> materialized) { return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, + public KTableX, V> aggregate(final Initializer initializer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, named, materialized)); } @Override - public ImprovedKTable, V> aggregate(final Initializer initializer, final Named named, + public KTableX, V> aggregate(final Initializer initializer, final Named named, final AutoMaterialized> materialized) { return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java similarity index 70% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java index 567a6586..7a33a2e7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java @@ -41,91 +41,91 @@ * @param type of keys * @param type of values */ -public interface ImprovedTimeWindowedKStream extends TimeWindowedKStream { +public interface TimeWindowedKStreamX extends TimeWindowedKStream { @Override - ImprovedKTable, Long> count(); + KTableX, Long> count(); @Override - ImprovedKTable, Long> count(Named named); + KTableX, Long> count(Named named); @Override - ImprovedKTable, Long> count(Materialized> materialized); + KTableX, Long> count(Materialized> materialized); /** * @see #count(Materialized) */ - ImprovedKTable, Long> count(AutoMaterialized> materialized); + KTableX, Long> count(AutoMaterialized> materialized); @Override - ImprovedKTable, Long> count(Named named, + KTableX, Long> count(Named named, Materialized> materialized); /** * @see #count(Named, Materialized) */ - ImprovedKTable, Long> count(Named named, + KTableX, Long> count(Named named, AutoMaterialized> materialized); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Named named); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Materialized) */ - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, AutoMaterialized> materialized); @Override - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Named named, Materialized> materialized); /** * @see #aggregate(Initializer, Aggregator, Named, Materialized) */ - ImprovedKTable, VR> aggregate(Initializer initializer, + KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Named named, AutoMaterialized> materialized); @Override - ImprovedKTable, V> reduce(Reducer reducer); + KTableX, V> reduce(Reducer reducer); @Override - ImprovedKTable, V> reduce(Reducer reducer, Named named); + KTableX, V> reduce(Reducer reducer, Named named); @Override - ImprovedKTable, V> reduce(Reducer reducer, + KTableX, V> reduce(Reducer reducer, Materialized> materialized); /** * @see #reduce(Reducer, Materialized) */ - ImprovedKTable, V> reduce(Reducer reducer, + KTableX, V> reduce(Reducer reducer, AutoMaterialized> materialized); @Override - ImprovedKTable, V> reduce(Reducer reducer, Named named, + KTableX, V> reduce(Reducer reducer, Named named, Materialized> materialized); /** * @see #reduce(Reducer, Named, Materialized) */ - ImprovedKTable, V> reduce(Reducer reducer, Named named, + KTableX, V> reduce(Reducer reducer, Named named, AutoMaterialized> materialized); @Override - ImprovedTimeWindowedKStream emitStrategy(EmitStrategy emitStrategy); + TimeWindowedKStreamX emitStrategy(EmitStrategy emitStrategy); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java similarity index 76% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java index 72e6d524..b7aba96b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedTimeWindowedStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java @@ -38,121 +38,121 @@ import org.apache.kafka.streams.state.WindowStore; @RequiredArgsConstructor -class ImprovedTimeWindowedStreamImpl implements ImprovedTimeWindowedKStream { +class TimeWindowedStreamXImpl implements TimeWindowedKStreamX { private final @NonNull TimeWindowedKStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKTable, Long> count() { + public KTableX, Long> count() { return this.context.wrap(this.wrapped.count()); } @Override - public ImprovedKTable, Long> count(final Named named) { + public KTableX, Long> count(final Named named) { return this.context.wrap(this.wrapped.count(named)); } @Override - public ImprovedKTable, Long> count( + public KTableX, Long> count( final Materialized> materialized) { return this.context.wrap(this.wrapped.count(materialized)); } @Override - public ImprovedKTable, Long> count( + public KTableX, Long> count( final AutoMaterialized> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, Long> count(final Named named, + public KTableX, Long> count(final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.count(named, materialized)); } @Override - public ImprovedKTable, Long> count(final Named named, + public KTableX, Long> count(final Named named, final AutoMaterialized> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, named)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.aggregate(initializer, aggregator, materialized)); } @Override - public ImprovedKTable, VR> aggregate(final Initializer initializer, + public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, final AutoMaterialized> materialized) { return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer) { + public KTableX, V> reduce(final Reducer reducer) { return this.context.wrap(this.wrapped.reduce(reducer)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, final Named named) { + public KTableX, V> reduce(final Reducer reducer, final Named named) { return this.context.wrap(this.wrapped.reduce(reducer, named)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, + public KTableX, V> reduce(final Reducer reducer, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, materialized)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, + public KTableX, V> reduce(final Reducer reducer, final AutoMaterialized> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + public KTableX, V> reduce(final Reducer reducer, final Named named, final Materialized> materialized) { return this.context.wrap(this.wrapped.reduce(reducer, named, materialized)); } @Override - public ImprovedKTable, V> reduce(final Reducer reducer, final Named named, + public KTableX, V> reduce(final Reducer reducer, final Named named, final AutoMaterialized> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } @Override - public ImprovedTimeWindowedKStream emitStrategy(final EmitStrategy emitStrategy) { + public TimeWindowedKStreamX emitStrategy(final EmitStrategy emitStrategy) { return this.context.wrap(this.wrapped.emitStrategy(emitStrategy)); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java index 73a89444..162d2918 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java @@ -875,7 +875,7 @@ void shouldConvertToTable() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input"); - final ImprovedKTable table = input.toTable(); + final KTableX table = input.toTable(); table.toStream().to("output"); } }; @@ -899,7 +899,7 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKTable table = input.toTable( + final KTableX table = input.toTable( AutoMaterialized.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); table.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -1394,8 +1394,8 @@ void shouldGroupByKey() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input"); - final ImprovedKGroupedStream grouped = input.groupByKey(); - final ImprovedKTable count = grouped.count(); + final KGroupedStreamX grouped = input.groupByKey(); + final KTableX count = grouped.count(); count.toStream().to("output"); } }; @@ -1424,10 +1424,10 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKGroupedStream grouped = input.groupByKey( + final KGroupedStreamX grouped = input.groupByKey( AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKTable count = grouped.count(); + final KTableX count = grouped.count(); count.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } @@ -1458,8 +1458,8 @@ void shouldGroupBy() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input"); - final ImprovedKGroupedStream grouped = input.groupBy((k, v) -> v); - final ImprovedKTable count = grouped.count(); + final KGroupedStreamX grouped = input.groupBy((k, v) -> v); + final KTableX count = grouped.count(); count.toStream().to("output"); } }; @@ -1488,10 +1488,10 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKGroupedStream grouped = input.groupBy((k, v) -> v, + final KGroupedStreamX grouped = input.groupBy((k, v) -> v, AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKTable count = grouped.count(); + final KTableX count = grouped.count(); count.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java similarity index 90% rename from streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java rename to streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index f3998db5..dc323b58 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKGroupedStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -30,7 +30,7 @@ import org.apache.kafka.common.serialization.Serdes; import org.junit.jupiter.api.Test; -class ImprovedKGroupedStreamTest { +class KGroupedStreamXTest { @Test void shouldReduce() { @@ -38,8 +38,8 @@ void shouldReduce() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input"); - final ImprovedKGroupedStream grouped = input.groupByKey(); - final ImprovedKTable reduced = grouped.reduce((value1, value2) -> value1 + value2); + final KGroupedStreamX grouped = input.groupByKey(); + final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2); reduced.toStream().to("output"); } }; @@ -67,10 +67,10 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKGroupedStream grouped = input.groupByKey( + final KGroupedStreamX grouped = input.groupByKey( AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKTable reduced = grouped.reduce(Long::sum, + final KTableX reduced = grouped.reduce(Long::sum, AutoMaterialized.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); reduced.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -103,8 +103,8 @@ void shouldAggregate() { @Override public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input"); - final ImprovedKGroupedStream grouped = input.groupByKey(); - final ImprovedKTable aggregated = + final KGroupedStreamX grouped = input.groupByKey(); + final KTableX aggregated = grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value); aggregated.toStream().to("output"); } @@ -133,10 +133,10 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKGroupedStream grouped = input.groupByKey( + final KGroupedStreamX grouped = input.groupByKey( AutoGrouped.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKTable aggregated = + final KTableX aggregated = grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, AutoMaterialized.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java index cd001e41..c3e1c227 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java @@ -25,7 +25,7 @@ package com.bakdata.kafka.test_applications; import com.bakdata.kafka.ImprovedKStream; -import com.bakdata.kafka.ImprovedKTable; +import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -51,7 +51,7 @@ public void buildTopology(final TopologyBuilder builder) { input.to(THROUGH_TOPIC); final ImprovedKStream through = builder.stream(THROUGH_TOPIC); - final ImprovedKTable, TestRecord> reduce = through + final KTableX, TestRecord> reduce = through .groupByKey() .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMillis(5L))) .reduce((a, b) -> a); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java index 3418860f..e4f82f20 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java @@ -25,7 +25,7 @@ package com.bakdata.kafka.test_applications; import com.bakdata.kafka.ImprovedKStream; -import com.bakdata.kafka.ImprovedKTable; +import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -46,7 +46,7 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream textLines = builder.streamInput(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); - final ImprovedKTable wordCounts = textLines + final KTableX wordCounts = textLines .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) .groupBy((key, word) -> word) .count(Materialized.as("counts")); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java index 7d737b1c..c9d35d04 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java @@ -25,7 +25,7 @@ package com.bakdata.kafka.test_applications; import com.bakdata.kafka.ImprovedKStream; -import com.bakdata.kafka.ImprovedKTable; +import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -47,7 +47,7 @@ public void buildTopology(final TopologyBuilder builder) { final ImprovedKStream textLines = builder.streamInputPattern(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); - final ImprovedKTable wordCounts = textLines + final KTableX wordCounts = textLines .flatMapValues(value -> Arrays.asList(pattern.split(value.toLowerCase()))) .groupBy((key, word) -> word) .count(Materialized.as("counts")); From f1d9198620bf3e749e22b2b2d747f52582d65c20 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 17:56:43 +0100 Subject: [PATCH 30/72] Add docs --- .../com/bakdata/kafka/ImprovedKStream.java | 413 ++++++++++++++++++ 1 file changed, 413 insertions(+) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java index 7ec8aadb..45195b4b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java @@ -82,9 +82,30 @@ public interface ImprovedKStream extends KStream { ImprovedKStream map( KeyValueMapper> mapper); + /** + * Transform each record of the input stream into a new record in the output stream. Errors in the mapper are + * captured + * @param mapper a {@link KeyValueMapper} that computes a new output record + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #map(KeyValueMapper) + * @see ErrorCapturingKeyValueMapper#captureErrors(KeyValueMapper) + */ KErrorStream mapCapturingErrors( KeyValueMapper> mapper); + /** + * Transform each record of the input stream into a new record in the output stream. Errors in the mapper are + * captured + * @param mapper a {@link KeyValueMapper} that computes a new output record + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #map(KeyValueMapper) + * @see ErrorCapturingKeyValueMapper#captureErrors(KeyValueMapper, java.util.function.Predicate) + */ KErrorStream mapCapturingErrors( KeyValueMapper> mapper, java.util.function.Predicate errorFilter); @@ -93,9 +114,32 @@ KErrorStream mapCapturingErrors( ImprovedKStream map( KeyValueMapper> mapper, Named named); + /** + * Transform each record of the input stream into a new record in the output stream. Errors in the mapper are + * captured + * @param mapper a {@link KeyValueMapper} that computes a new output record + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #map(KeyValueMapper, Named) + * @see ErrorCapturingKeyValueMapper#captureErrors(KeyValueMapper) + */ KErrorStream mapCapturingErrors( KeyValueMapper> mapper, Named named); + /** + * Transform each record of the input stream into a new record in the output stream. Errors in the mapper are + * captured + * @param mapper a {@link KeyValueMapper} that computes a new output record + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #map(KeyValueMapper, Named) + * @see ErrorCapturingKeyValueMapper#captureErrors(KeyValueMapper, java.util.function.Predicate) + */ KErrorStream mapCapturingErrors( KeyValueMapper> mapper, java.util.function.Predicate errorFilter, Named named); @@ -103,25 +147,84 @@ KErrorStream mapCapturingErrors( @Override ImprovedKStream mapValues(ValueMapper mapper); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes a new output value + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapper) + * @see ErrorCapturingValueMapper#captureErrors(ValueMapper) + */ KErrorStream mapValuesCapturingErrors(ValueMapper mapper); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes a new output value + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapper) + * @see ErrorCapturingValueMapper#captureErrors(ValueMapper, java.util.function.Predicate) + */ KErrorStream mapValuesCapturingErrors(ValueMapper mapper, java.util.function.Predicate errorFilter); @Override ImprovedKStream mapValues(ValueMapper mapper, Named named); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes a new output value + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapper, Named) + * @see ErrorCapturingValueMapper#captureErrors(ValueMapper) + */ KErrorStream mapValuesCapturingErrors(ValueMapper mapper, Named named); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes a new output value + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapper, Named) + * @see ErrorCapturingValueMapper#captureErrors(ValueMapper, java.util.function.Predicate) + */ KErrorStream mapValuesCapturingErrors(ValueMapper mapper, java.util.function.Predicate errorFilter, Named named); @Override ImprovedKStream mapValues(ValueMapperWithKey mapper); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes a new output value + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapperWithKey) + * @see ErrorCapturingValueMapperWithKey#captureErrors(ValueMapperWithKey) + */ KErrorStream mapValuesCapturingErrors( ValueMapperWithKey mapper); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes a new output value + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapperWithKey) + * @see ErrorCapturingValueMapperWithKey#captureErrors(ValueMapperWithKey, java.util.function.Predicate) + */ KErrorStream mapValuesCapturingErrors( ValueMapperWithKey mapper, java.util.function.Predicate errorFilter); @@ -129,9 +232,30 @@ KErrorStream mapValuesCapturingErrors( @Override ImprovedKStream mapValues(ValueMapperWithKey mapper, Named named); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes a new output value + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapperWithKey, Named) + * @see ErrorCapturingValueMapperWithKey#captureErrors(ValueMapperWithKey) + */ KErrorStream mapValuesCapturingErrors( ValueMapperWithKey mapper, Named named); + /** + * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes a new output value + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #mapValues(ValueMapperWithKey, Named) + * @see ErrorCapturingValueMapperWithKey#captureErrors(ValueMapperWithKey, java.util.function.Predicate) + */ KErrorStream mapValuesCapturingErrors( ValueMapperWithKey mapper, java.util.function.Predicate errorFilter, Named named); @@ -140,9 +264,32 @@ KErrorStream mapValuesCapturingErrors( ImprovedKStream flatMap( KeyValueMapper>> mapper); + /** + * Transform each record of the input stream into zero or more records in the output stream. Errors in the mapper + * are captured + * @param mapper a {@link KeyValueMapper} that computes the new output records + * @return a {@link KErrorStream} that contains more or less records with new key and value as well as captured + * errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #flatMap(KeyValueMapper) + * @see ErrorCapturingFlatKeyValueMapper#captureErrors(KeyValueMapper) + */ KErrorStream flatMapCapturingErrors( KeyValueMapper>> mapper); + /** + * Transform each record of the input stream into zero or more records in the output stream. Errors in the mapper + * are captured + * @param mapper a {@link KeyValueMapper} that computes the new output records + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains more or less records with new key and value as well as captured + * errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #flatMap(KeyValueMapper) + * @see ErrorCapturingFlatKeyValueMapper#captureErrors(KeyValueMapper, java.util.function.Predicate) + */ KErrorStream flatMapCapturingErrors( KeyValueMapper>> mapper, java.util.function.Predicate errorFilter); @@ -152,10 +299,35 @@ ImprovedKStream flatMap( KeyValueMapper>> mapper, Named named); + /** + * Transform each record of the input stream into zero or more records in the output stream. Errors in the mapper + * are captured + * @param mapper a {@link KeyValueMapper} that computes the new output records + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains more or less records with new key and value as well as captured + * errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #flatMap(KeyValueMapper, Named) + * @see ErrorCapturingFlatKeyValueMapper#captureErrors(KeyValueMapper) + */ KErrorStream flatMapCapturingErrors( KeyValueMapper>> mapper, Named named); + /** + * Transform each record of the input stream into zero or more records in the output stream. Errors in the mapper + * are captured + * @param mapper a {@link KeyValueMapper} that computes the new output records + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains more or less records with new key and value as well as captured + * errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #flatMap(KeyValueMapper, Named) + * @see ErrorCapturingFlatKeyValueMapper#captureErrors(KeyValueMapper, java.util.function.Predicate) + */ KErrorStream flatMapCapturingErrors( KeyValueMapper>> mapper, java.util.function.Predicate errorFilter, Named named); @@ -163,9 +335,32 @@ KErrorStream flatMapCapturingErrors( @Override ImprovedKStream flatMapValues(ValueMapper> mapper); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes the new output values + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapper) + * @see ErrorCapturingFlatValueMapper#captureErrors(ValueMapper) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapper> mapper); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes the new output values + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapper) + * @see ErrorCapturingFlatValueMapper#captureErrors(ValueMapper, java.util.function.Predicate) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapper> mapper, java.util.function.Predicate errorFilter); @@ -174,10 +369,35 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKStream flatMapValues(ValueMapper> mapper, Named named); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes the new output values + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapper, Named) + * @see ErrorCapturingFlatValueMapper#captureErrors(ValueMapper,) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapper> mapper, Named named); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapper} that computes the new output values + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapper, Named) + * @see ErrorCapturingFlatValueMapper#captureErrors(ValueMapper, java.util.function.Predicate) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapper> mapper, java.util.function.Predicate errorFilter, Named named); @@ -186,9 +406,32 @@ KErrorStream flatMapValuesCapturingErrors( ImprovedKStream flatMapValues( ValueMapperWithKey> mapper); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes the new output values + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapperWithKey) + * @see ErrorCapturingFlatValueMapperWithKey#captureErrors(ValueMapperWithKey) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapperWithKey> mapper); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes the new output values + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapperWithKey) + * @see ErrorCapturingFlatValueMapperWithKey#captureErrors(ValueMapperWithKey, java.util.function.Predicate) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapperWithKey> mapper, java.util.function.Predicate errorFilter); @@ -198,9 +441,34 @@ ImprovedKStream flatMapValues( ValueMapperWithKey> mapper, Named named); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes the new output values + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapperWithKey, Named) + * @see ErrorCapturingFlatValueMapperWithKey#captureErrors(ValueMapperWithKey) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapperWithKey> mapper, Named named); + /** + * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values + * with + * the same key in the new stream. Errors in the mapper are captured + * @param mapper a {@link ValueMapperWithKey} that computes the new output values + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains more or less records with unmodified keys and new values as well + * as captured errors + * @param the value type of the result stream + * @see #flatMapValues(ValueMapperWithKey, Named) + * @see ErrorCapturingFlatValueMapperWithKey#captureErrors(ValueMapperWithKey, java.util.function.Predicate) + */ KErrorStream flatMapValuesCapturingErrors( ValueMapperWithKey> mapper, java.util.function.Predicate errorFilter, Named named); @@ -256,22 +524,67 @@ KErrorStream flatMapValuesCapturingErrors( */ void to(TopicNameExtractor topicExtractor, AutoProduced produced); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic()} + * @see #to(String) + */ void toOutputTopic(); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic()} + * @param produced define optional parameters for materializing the stream + * @see #to(String, Produced) + */ void toOutputTopic(Produced produced); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic()} + * @param produced define optional parameters for materializing the stream + * @see #to(String, Produced) + */ void toOutputTopic(AutoProduced produced); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic(String)} + * @param label label of output topic + * @see #to(String) + */ void toOutputTopic(String label); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic(String)} + * @param label label of output topic + * @param produced define optional parameters for materializing the stream + * @see #to(String, Produced) + */ void toOutputTopic(String label, Produced produced); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic(String)} + * @param label label of output topic + * @param produced define optional parameters for materializing the stream + * @see #to(String, Produced) + */ void toOutputTopic(String label, AutoProduced produced); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getErrorTopic()} + * @see #to(String) + */ void toErrorTopic(); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getErrorTopic()} + * @param produced define optional parameters for materializing the stream + * @see #to(String, Produced) + */ void toErrorTopic(Produced produced); + /** + * Materialize {@link KStream} to {@link StreamsTopicConfig#getErrorTopic()} + * @param produced define optional parameters for materializing the stream + * @see #to(String, Produced) + */ void toErrorTopic(AutoProduced produced); @Override @@ -581,10 +894,33 @@ ImprovedKStream process( ProcessorSupplier processorSupplier, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.Processor}. Errors in the mapper are captured + * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.Processor} + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #process(ProcessorSupplier, String...) + * @see ErrorCapturingProcessor#captureErrors(ProcessorSupplier) + */ KErrorStream processCapturingErrors( ProcessorSupplier processorSupplier, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.Processor}. Errors in the mapper are captured + * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.Processor} + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #process(ProcessorSupplier, String...) + * @see ErrorCapturingProcessor#captureErrors(ProcessorSupplier, java.util.function.Predicate) + */ KErrorStream processCapturingErrors( ProcessorSupplier processorSupplier, java.util.function.Predicate errorFilter, @@ -595,10 +931,35 @@ ImprovedKStream process( ProcessorSupplier processorSupplier, Named named, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.Processor}. Errors in the mapper are captured + * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.Processor} + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #process(ProcessorSupplier, Named, String...) + * @see ErrorCapturingProcessor#captureErrors(ProcessorSupplier) + */ KErrorStream processCapturingErrors( ProcessorSupplier processorSupplier, Named named, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.Processor}. Errors in the mapper are captured + * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.Processor} + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors + * @param the key type of the result stream + * @param the value type of the result stream + * @see #process(ProcessorSupplier, Named, String...) + * @see ErrorCapturingProcessor#captureErrors(ProcessorSupplier, java.util.function.Predicate) + */ KErrorStream processCapturingErrors( ProcessorSupplier processorSupplier, java.util.function.Predicate errorFilter, @@ -609,10 +970,35 @@ ImprovedKStream processValues( FixedKeyProcessorSupplier processorSupplier, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor}. Errors in the + * mapper are captured + * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #processValues(FixedKeyProcessorSupplier, String...) + * @see ErrorCapturingValueProcessor#captureErrors(FixedKeyProcessorSupplier) + */ KErrorStream processValuesCapturingErrors( FixedKeyProcessorSupplier processorSupplier, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor}. Errors in the + * mapper are captured + * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} + * @param errorFilter expression that filters errors which should be thrown and not captured + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #processValues(FixedKeyProcessorSupplier, String...) + * @see ErrorCapturingValueProcessor#captureErrors(FixedKeyProcessorSupplier, java.util.function.Predicate) + */ KErrorStream processValuesCapturingErrors( FixedKeyProcessorSupplier processorSupplier, java.util.function.Predicate errorFilter, @@ -623,10 +1009,37 @@ ImprovedKStream processValues( FixedKeyProcessorSupplier processorSupplier, Named named, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor}. Errors in the + * mapper are captured + * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #processValues(FixedKeyProcessorSupplier, Named, String...) + * @see ErrorCapturingValueProcessor#captureErrors(FixedKeyProcessorSupplier) + */ KErrorStream processValuesCapturingErrors( FixedKeyProcessorSupplier processorSupplier, Named named, String... stateStoreNames); + /** + * Process all records in this stream, one record at a time, by applying a + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor}. Errors in the + * mapper are captured + * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed + * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} + * @param errorFilter expression that filters errors which should be thrown and not captured + * @param named a {@link Named} config used to name the processor in the topology + * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured + * errors + * @param the value type of the result stream + * @see #processValues(FixedKeyProcessorSupplier, Named, String...) + * @see ErrorCapturingValueProcessor#captureErrors(FixedKeyProcessorSupplier, java.util.function.Predicate) + */ KErrorStream processValuesCapturingErrors( FixedKeyProcessorSupplier processorSupplier, java.util.function.Predicate errorFilter, From 210c6d745534c17bc2fd3ccf32ede792dd901da3 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 18:01:59 +0100 Subject: [PATCH 31/72] Rename --- README.md | 4 +- .../java/com/bakdata/kafka/CloseFlagApp.java | 2 +- .../kafka/test_applications/Mirror.java | 4 +- .../kafka/test_applications/WordCount.java | 4 +- .../bakdata/kafka/BranchedKStreamXImpl.java | 2 +- .../java/com/bakdata/kafka/KErrorStream.java | 18 +- .../{ImprovedKStream.java => KStreamX.java} | 170 +++++++-------- ...ovedKStreamImpl.java => KStreamXImpl.java} | 204 +++++++++--------- .../main/java/com/bakdata/kafka/KTableX.java | 8 +- .../java/com/bakdata/kafka/KTableXImpl.java | 8 +- .../bakdata/kafka/KeyValueKErrorStream.java | 10 +- .../com/bakdata/kafka/StreamsContext.java | 14 +- .../com/bakdata/kafka/TopologyBuilder.java | 42 ++-- .../com/bakdata/kafka/ValueKErrorStream.java | 10 +- .../bakdata/kafka/KGroupedStreamXTest.java | 10 +- ...ovedKStreamTest.java => KStreamXTest.java} | 148 ++++++------- .../bakdata/kafka/TopologyBuilderTest.java | 18 +- .../kafka/integration/StreamsRunnerTest.java | 4 +- .../ComplexTopologyApplication.java | 6 +- .../test_applications/LabeledInputTopics.java | 4 +- .../kafka/test_applications/Mirror.java | 4 +- .../test_applications/MirrorKeyWithAvro.java | 4 +- .../MirrorValueWithAvro.java | 4 +- .../MirrorWithNonDefaultSerde.java | 4 +- .../kafka/test_applications/WordCount.java | 4 +- .../test_applications/WordCountPattern.java | 4 +- 26 files changed, 357 insertions(+), 357 deletions(-) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKStream.java => KStreamX.java} (88%) rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{ImprovedKStreamImpl.java => KStreamXImpl.java} (84%) rename streams-bootstrap-core/src/test/java/com/bakdata/kafka/{ImprovedKStreamTest.java => KStreamXTest.java} (90%) diff --git a/README.md b/README.md index e8ae80ae..4bcf443b 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Create a subclass of `KafkaStreamsApplication` and implement the abstract method and `getUniqueAppId()`. You can define the topology of your application in `buildTopology()`. ```java -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.KafkaStreamsApplication; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -76,7 +76,7 @@ public class MyStreamsApplication extends KafkaStreamsApplication { return new StreamsApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); // your topology diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java index acf7aaee..3183feb5 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/CloseFlagApp.java @@ -48,7 +48,7 @@ public StreamsApp createApp() { return new StreamsApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.toOutputTopic(); } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java index 8833fae6..846b62a8 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/Mirror.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -36,7 +36,7 @@ public class Mirror implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.toOutputTopic(); } diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java index e4f82f20..c4047d96 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/test_applications/WordCount.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -43,7 +43,7 @@ public class WordCount implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream textLines = builder.streamInput(); + final KStreamX textLines = builder.streamInput(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); final KTableX wordCounts = textLines diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java index 7a005540..e210fb9e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java @@ -72,7 +72,7 @@ private Map> wrap(final Map wrapValue(final Entry> entry) { + private KStreamX wrapValue(final Entry> entry) { return this.context.wrap(entry.getValue()); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java index 9af54689..78af3d9d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KErrorStream.java @@ -27,11 +27,11 @@ import org.apache.kafka.streams.kstream.Named; /** - * {@link ImprovedKStream} that contains successfully processed records and errors of a previous operation - * @param type of keys in the original {@link ImprovedKStream} - * @param type of values in the original {@link ImprovedKStream} - * @param type of keys in the processed {@link ImprovedKStream} - * @param type of values in the processed {@link ImprovedKStream} + * {@link KStreamX} that contains successfully processed records and errors of a previous operation + * @param type of keys in the original {@link KStreamX} + * @param type of values in the original {@link KStreamX} + * @param type of keys in the processed {@link KStreamX} + * @param type of values in the processed {@link KStreamX} */ public interface KErrorStream { @@ -39,25 +39,25 @@ public interface KErrorStream { * Get the stream of successfully processed values * @return stream of processed values */ - ImprovedKStream values(); + KStreamX values(); /** * Get the stream of successfully processed values * @param named name of the processor * @return stream of processed values */ - ImprovedKStream values(Named named); + KStreamX values(Named named); /** * Get the stream of errors that occurred during processing * @return stream of errors */ - ImprovedKStream> errors(); + KStreamX> errors(); /** * Get the stream of errors that occurred during processing * @param named name of the processor * @return stream of errors */ - ImprovedKStream> errors(Named named); + KStreamX> errors(Named named); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java similarity index 88% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java index 45195b4b..c44c2a3b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java @@ -58,28 +58,28 @@ * @param type of keys * @param type of values */ -public interface ImprovedKStream extends KStream { +public interface KStreamX extends KStream { @Override - ImprovedKStream filter(Predicate predicate); + KStreamX filter(Predicate predicate); @Override - ImprovedKStream filter(Predicate predicate, Named named); + KStreamX filter(Predicate predicate, Named named); @Override - ImprovedKStream filterNot(Predicate predicate); + KStreamX filterNot(Predicate predicate); @Override - ImprovedKStream filterNot(Predicate predicate, Named named); + KStreamX filterNot(Predicate predicate, Named named); @Override - ImprovedKStream selectKey(KeyValueMapper mapper); + KStreamX selectKey(KeyValueMapper mapper); @Override - ImprovedKStream selectKey(KeyValueMapper mapper, Named named); + KStreamX selectKey(KeyValueMapper mapper, Named named); @Override - ImprovedKStream map( + KStreamX map( KeyValueMapper> mapper); /** @@ -111,7 +111,7 @@ KErrorStream mapCapturingErrors( java.util.function.Predicate errorFilter); @Override - ImprovedKStream map( + KStreamX map( KeyValueMapper> mapper, Named named); /** @@ -145,7 +145,7 @@ KErrorStream mapCapturingErrors( java.util.function.Predicate errorFilter, Named named); @Override - ImprovedKStream mapValues(ValueMapper mapper); + KStreamX mapValues(ValueMapper mapper); /** * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured @@ -172,7 +172,7 @@ KErrorStream mapValuesCapturingErrors(ValueMapper errorFilter); @Override - ImprovedKStream mapValues(ValueMapper mapper, Named named); + KStreamX mapValues(ValueMapper mapper, Named named); /** * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured @@ -201,7 +201,7 @@ KErrorStream mapValuesCapturingErrors(ValueMapper errorFilter, Named named); @Override - ImprovedKStream mapValues(ValueMapperWithKey mapper); + KStreamX mapValues(ValueMapperWithKey mapper); /** * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured @@ -230,7 +230,7 @@ KErrorStream mapValuesCapturingErrors( java.util.function.Predicate errorFilter); @Override - ImprovedKStream mapValues(ValueMapperWithKey mapper, Named named); + KStreamX mapValues(ValueMapperWithKey mapper, Named named); /** * Transform the value of each input record into a new value of the output record. Errors in the mapper are captured @@ -261,7 +261,7 @@ KErrorStream mapValuesCapturingErrors( java.util.function.Predicate errorFilter, Named named); @Override - ImprovedKStream flatMap( + KStreamX flatMap( KeyValueMapper>> mapper); /** @@ -295,7 +295,7 @@ KErrorStream flatMapCapturingErrors( java.util.function.Predicate errorFilter); @Override - ImprovedKStream flatMap( + KStreamX flatMap( KeyValueMapper>> mapper, Named named); @@ -333,7 +333,7 @@ KErrorStream flatMapCapturingErrors( java.util.function.Predicate errorFilter, Named named); @Override - ImprovedKStream flatMapValues(ValueMapper> mapper); + KStreamX flatMapValues(ValueMapper> mapper); /** * Create a new {@link KStream} by transforming the value of each record in this stream into zero or more values @@ -366,7 +366,7 @@ KErrorStream flatMapValuesCapturingErrors( java.util.function.Predicate errorFilter); @Override - ImprovedKStream flatMapValues(ValueMapper> mapper, + KStreamX flatMapValues(ValueMapper> mapper, Named named); /** @@ -379,7 +379,7 @@ ImprovedKStream flatMapValues(ValueMapper the value type of the result stream * @see #flatMapValues(ValueMapper, Named) - * @see ErrorCapturingFlatValueMapper#captureErrors(ValueMapper,) + * @see ErrorCapturingFlatValueMapper#captureErrors(ValueMapper) */ KErrorStream flatMapValuesCapturingErrors( ValueMapper> mapper, @@ -403,7 +403,7 @@ KErrorStream flatMapValuesCapturingErrors( java.util.function.Predicate errorFilter, Named named); @Override - ImprovedKStream flatMapValues( + KStreamX flatMapValues( ValueMapperWithKey> mapper); /** @@ -437,7 +437,7 @@ KErrorStream flatMapValuesCapturingErrors( java.util.function.Predicate errorFilter); @Override - ImprovedKStream flatMapValues( + KStreamX flatMapValues( ValueMapperWithKey> mapper, Named named); @@ -474,16 +474,16 @@ KErrorStream flatMapValuesCapturingErrors( java.util.function.Predicate errorFilter, Named named); @Override - ImprovedKStream peek(ForeachAction action); + KStreamX peek(ForeachAction action); @Override - ImprovedKStream peek(ForeachAction action, Named named); + KStreamX peek(ForeachAction action, Named named); @Override - ImprovedKStream[] branch(Named named, Predicate... predicates); + KStreamX[] branch(Named named, Predicate... predicates); @Override - ImprovedKStream[] branch(Predicate... predicates); + KStreamX[] branch(Predicate... predicates); @Override BranchedKStreamX split(); @@ -492,27 +492,27 @@ KErrorStream flatMapValuesCapturingErrors( BranchedKStreamX split(Named named); @Override - ImprovedKStream merge(KStream stream); + KStreamX merge(KStream stream); @Override - ImprovedKStream merge(KStream stream, Named named); + KStreamX merge(KStream stream, Named named); @Override - ImprovedKStream through(String topic); + KStreamX through(String topic); @Override - ImprovedKStream through(String topic, Produced produced); + KStreamX through(String topic, Produced produced); @Override - ImprovedKStream repartition(); + KStreamX repartition(); @Override - ImprovedKStream repartition(Repartitioned repartitioned); + KStreamX repartition(Repartitioned repartitioned); /** * @see #repartition(Repartitioned) */ - ImprovedKStream repartition(AutoRepartitioned repartitioned); + KStreamX repartition(AutoRepartitioned repartitioned); /** * @see #to(String, Produced) @@ -634,263 +634,263 @@ KGroupedStreamX groupBy(KeyValueMapper key KGroupedStreamX groupByKey(AutoGrouped grouped); @Override - ImprovedKStream join(KStream otherStream, + KStreamX join(KStream otherStream, ValueJoiner joiner, JoinWindows windows); @Override - ImprovedKStream join(KStream otherStream, + KStreamX join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows); @Override - ImprovedKStream join(KStream otherStream, + KStreamX join(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); /** * @see #join(KStream, ValueJoiner, JoinWindows, StreamJoined) */ - ImprovedKStream join(KStream otherStream, + KStreamX join(KStream otherStream, ValueJoiner joiner, JoinWindows windows, AutoStreamJoined streamJoined); @Override - ImprovedKStream join(KStream otherStream, + KStreamX join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); /** * @see #join(KStream, ValueJoinerWithKey, JoinWindows, StreamJoined) */ - ImprovedKStream join(KStream otherStream, + KStreamX join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, AutoStreamJoined streamJoined); @Override - ImprovedKStream leftJoin(KStream otherStream, + KStreamX leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows); @Override - ImprovedKStream leftJoin(KStream otherStream, + KStreamX leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows); @Override - ImprovedKStream leftJoin(KStream otherStream, + KStreamX leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); /** * @see #leftJoin(KStream, ValueJoiner, JoinWindows, StreamJoined) */ - ImprovedKStream leftJoin(KStream otherStream, + KStreamX leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, AutoStreamJoined streamJoined); @Override - ImprovedKStream leftJoin(KStream otherStream, + KStreamX leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); /** * @see #leftJoin(KStream, ValueJoinerWithKey, JoinWindows, StreamJoined) */ - ImprovedKStream leftJoin(KStream otherStream, + KStreamX leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, AutoStreamJoined streamJoined); @Override - ImprovedKStream outerJoin(KStream otherStream, + KStreamX outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows); @Override - ImprovedKStream outerJoin(KStream otherStream, + KStreamX outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows); @Override - ImprovedKStream outerJoin(KStream otherStream, + KStreamX outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, StreamJoined streamJoined); /** * @see #outerJoin(KStream, ValueJoiner, JoinWindows, StreamJoined) */ - ImprovedKStream outerJoin(KStream otherStream, + KStreamX outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, AutoStreamJoined streamJoined); @Override - ImprovedKStream outerJoin(KStream otherStream, + KStreamX outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, StreamJoined streamJoined); /** * @see #outerJoin(KStream, ValueJoinerWithKey, JoinWindows, StreamJoined) */ - ImprovedKStream outerJoin(KStream otherStream, + KStreamX outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, AutoStreamJoined streamJoined); @Override - ImprovedKStream join(KTable table, ValueJoiner joiner); + KStreamX join(KTable table, ValueJoiner joiner); @Override - ImprovedKStream join(KTable table, + KStreamX join(KTable table, ValueJoinerWithKey joiner); @Override - ImprovedKStream join(KTable table, ValueJoiner joiner, + KStreamX join(KTable table, ValueJoiner joiner, Joined joined); /** * @see #join(KTable, ValueJoiner, Joined) */ - ImprovedKStream join(KTable table, ValueJoiner joiner, + KStreamX join(KTable table, ValueJoiner joiner, AutoJoined joined); @Override - ImprovedKStream join(KTable table, + KStreamX join(KTable table, ValueJoinerWithKey joiner, Joined joined); /** * @see #join(KTable, ValueJoinerWithKey, Joined) */ - ImprovedKStream join(KTable table, + KStreamX join(KTable table, ValueJoinerWithKey joiner, AutoJoined joined); @Override - ImprovedKStream leftJoin(KTable table, + KStreamX leftJoin(KTable table, ValueJoiner joiner); @Override - ImprovedKStream leftJoin(KTable table, + KStreamX leftJoin(KTable table, ValueJoinerWithKey joiner); @Override - ImprovedKStream leftJoin(KTable table, + KStreamX leftJoin(KTable table, ValueJoiner joiner, Joined joined); /** * @see #leftJoin(KTable, ValueJoiner, Joined) */ - ImprovedKStream leftJoin(KTable table, + KStreamX leftJoin(KTable table, ValueJoiner joiner, AutoJoined joined); @Override - ImprovedKStream leftJoin(KTable table, + KStreamX leftJoin(KTable table, ValueJoinerWithKey joiner, Joined joined); /** * @see #leftJoin(KTable, ValueJoinerWithKey, Joined) */ - ImprovedKStream leftJoin(KTable table, + KStreamX leftJoin(KTable table, ValueJoinerWithKey joiner, AutoJoined joined); @Override - ImprovedKStream join(GlobalKTable globalTable, + KStreamX join(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoiner joiner); @Override - ImprovedKStream join(GlobalKTable globalTable, + KStreamX join(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey joiner); @Override - ImprovedKStream join(GlobalKTable globalTable, + KStreamX join(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoiner joiner, Named named); @Override - ImprovedKStream join(GlobalKTable globalTable, + KStreamX join(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey joiner, Named named); @Override - ImprovedKStream leftJoin(GlobalKTable globalTable, + KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoiner valueJoiner); @Override - ImprovedKStream leftJoin(GlobalKTable globalTable, + KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey valueJoiner); @Override - ImprovedKStream leftJoin(GlobalKTable globalTable, + KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoiner valueJoiner, Named named); @Override - ImprovedKStream leftJoin(GlobalKTable globalTable, + KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey valueJoiner, Named named); @Override - ImprovedKStream transform( + KStreamX transform( TransformerSupplier> transformerSupplier, String... stateStoreNames); @Override - ImprovedKStream transform( + KStreamX transform( TransformerSupplier> transformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKStream flatTransform( + KStreamX flatTransform( TransformerSupplier>> transformerSupplier, String... stateStoreNames); @Override - ImprovedKStream flatTransform( + KStreamX flatTransform( TransformerSupplier>> transformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKStream transformValues( + KStreamX transformValues( ValueTransformerSupplier valueTransformerSupplier, String... stateStoreNames); @Override - ImprovedKStream transformValues( + KStreamX transformValues( ValueTransformerSupplier valueTransformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKStream transformValues( + KStreamX transformValues( ValueTransformerWithKeySupplier valueTransformerSupplier, String... stateStoreNames); @Override - ImprovedKStream transformValues( + KStreamX transformValues( ValueTransformerWithKeySupplier valueTransformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKStream flatTransformValues( + KStreamX flatTransformValues( ValueTransformerSupplier> valueTransformerSupplier, String... stateStoreNames); @Override - ImprovedKStream flatTransformValues( + KStreamX flatTransformValues( ValueTransformerSupplier> valueTransformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKStream flatTransformValues( + KStreamX flatTransformValues( ValueTransformerWithKeySupplier> valueTransformerSupplier, String... stateStoreNames); @Override - ImprovedKStream flatTransformValues( + KStreamX flatTransformValues( ValueTransformerWithKeySupplier> valueTransformerSupplier, Named named, String... stateStoreNames); @Override - ImprovedKStream process( + KStreamX process( ProcessorSupplier processorSupplier, String... stateStoreNames); @@ -927,7 +927,7 @@ KErrorStream processCapturingErrors( String... stateStoreNames); @Override - ImprovedKStream process( + KStreamX process( ProcessorSupplier processorSupplier, Named named, String... stateStoreNames); @@ -966,7 +966,7 @@ KErrorStream processCapturingErrors( Named named, String... stateStoreNames); @Override - ImprovedKStream processValues( + KStreamX processValues( FixedKeyProcessorSupplier processorSupplier, String... stateStoreNames); @@ -1005,7 +1005,7 @@ KErrorStream processValuesCapturingErrors( String... stateStoreNames); @Override - ImprovedKStream processValues( + KStreamX processValues( FixedKeyProcessorSupplier processorSupplier, Named named, String... stateStoreNames); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java similarity index 84% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java index 4d8a090c..f51685a6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ImprovedKStreamImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java @@ -59,45 +59,45 @@ import org.apache.kafka.streams.state.KeyValueStore; @RequiredArgsConstructor -class ImprovedKStreamImpl implements ImprovedKStream { +class KStreamXImpl implements KStreamX { @Getter(AccessLevel.PACKAGE) private final @NonNull KStream wrapped; private final @NonNull StreamsContext context; @Override - public ImprovedKStream filter(final Predicate predicate) { + public KStreamX filter(final Predicate predicate) { return this.context.wrap(this.wrapped.filter(predicate)); } @Override - public ImprovedKStream filter(final Predicate predicate, final Named named) { + public KStreamX filter(final Predicate predicate, final Named named) { return this.context.wrap(this.wrapped.filter(predicate, named)); } @Override - public ImprovedKStream filterNot(final Predicate predicate) { + public KStreamX filterNot(final Predicate predicate) { return this.context.wrap(this.wrapped.filterNot(predicate)); } @Override - public ImprovedKStream filterNot(final Predicate predicate, final Named named) { + public KStreamX filterNot(final Predicate predicate, final Named named) { return this.context.wrap(this.wrapped.filterNot(predicate, named)); } @Override - public ImprovedKStream selectKey(final KeyValueMapper mapper) { + public KStreamX selectKey(final KeyValueMapper mapper) { return this.context.wrap(this.wrapped.selectKey(mapper)); } @Override - public ImprovedKStream selectKey(final KeyValueMapper mapper, + public KStreamX selectKey(final KeyValueMapper mapper, final Named named) { return this.context.wrap(this.wrapped.selectKey(mapper, named)); } @Override - public ImprovedKStream map( + public KStreamX map( final KeyValueMapper> mapper) { return this.context.wrap(this.wrapped.map(mapper)); } @@ -116,7 +116,7 @@ public KErrorStream mapCapturingErrors( } @Override - public ImprovedKStream map( + public KStreamX map( final KeyValueMapper> mapper, final Named named) { return this.context.wrap(this.wrapped.map(mapper, named)); @@ -137,7 +137,7 @@ public KErrorStream mapCapturingErrors( } @Override - public ImprovedKStream mapValues(final ValueMapper mapper) { + public KStreamX mapValues(final ValueMapper mapper) { return this.context.wrap(this.wrapped.mapValues(mapper)); } @@ -154,7 +154,7 @@ public KErrorStream mapValuesCapturingErrors(final ValueMapper } @Override - public ImprovedKStream mapValues(final ValueMapper mapper, final Named named) { + public KStreamX mapValues(final ValueMapper mapper, final Named named) { return this.context.wrap(this.wrapped.mapValues(mapper, named)); } @@ -172,7 +172,7 @@ public KErrorStream mapValuesCapturingErrors(final ValueMapper } @Override - public ImprovedKStream mapValues(final ValueMapperWithKey mapper) { + public KStreamX mapValues(final ValueMapperWithKey mapper) { return this.context.wrap(this.wrapped.mapValues(mapper)); } @@ -191,7 +191,7 @@ public KErrorStream mapValuesCapturingErrors( } @Override - public ImprovedKStream mapValues(final ValueMapperWithKey mapper, + public KStreamX mapValues(final ValueMapperWithKey mapper, final Named named) { return this.context.wrap(this.wrapped.mapValues(mapper, named)); } @@ -212,7 +212,7 @@ public KErrorStream mapValuesCapturingErrors( } @Override - public ImprovedKStream flatMap( + public KStreamX flatMap( final KeyValueMapper>> mapper) { return this.context.wrap(this.wrapped.flatMap(mapper)); @@ -234,7 +234,7 @@ public KErrorStream flatMapCapturingErrors( } @Override - public ImprovedKStream flatMap( + public KStreamX flatMap( final KeyValueMapper>> mapper, final Named named) { @@ -259,7 +259,7 @@ public KErrorStream flatMapCapturingErrors( } @Override - public ImprovedKStream flatMapValues( + public KStreamX flatMapValues( final ValueMapper> mapper) { return this.context.wrap(this.wrapped.flatMapValues(mapper)); } @@ -279,7 +279,7 @@ public KErrorStream flatMapValuesCapturingErrors( } @Override - public ImprovedKStream flatMapValues( + public KStreamX flatMapValues( final ValueMapper> mapper, final Named named) { return this.context.wrap(this.wrapped.flatMapValues(mapper, named)); @@ -302,7 +302,7 @@ public KErrorStream flatMapValuesCapturingErrors( } @Override - public ImprovedKStream flatMapValues( + public KStreamX flatMapValues( final ValueMapperWithKey> mapper) { return this.context.wrap(this.wrapped.flatMapValues(mapper)); } @@ -322,7 +322,7 @@ public KErrorStream flatMapValuesCapturingErrors( } @Override - public ImprovedKStream flatMapValues( + public KStreamX flatMapValues( final ValueMapperWithKey> mapper, final Named named) { return this.context.wrap(this.wrapped.flatMapValues(mapper, named)); @@ -360,27 +360,27 @@ public void foreach(final ForeachAction action, final Name } @Override - public ImprovedKStream peek(final ForeachAction action) { + public KStreamX peek(final ForeachAction action) { return this.context.wrap(this.wrapped.peek(action)); } @Override - public ImprovedKStream peek(final ForeachAction action, final Named named) { + public KStreamX peek(final ForeachAction action, final Named named) { return this.context.wrap(this.wrapped.peek(action, named)); } @Override - public ImprovedKStream[] branch(final Predicate... predicates) { + public KStreamX[] branch(final Predicate... predicates) { return Arrays.stream(this.wrapped.branch(predicates)) .map(this.context::wrap) - .toArray(ImprovedKStream[]::new); + .toArray(KStreamX[]::new); } @Override - public ImprovedKStream[] branch(final Named named, final Predicate... predicates) { + public KStreamX[] branch(final Named named, final Predicate... predicates) { return Arrays.stream(this.wrapped.branch(named, predicates)) .map(this.context::wrap) - .toArray(ImprovedKStream[]::new); + .toArray(KStreamX[]::new); } @Override @@ -394,39 +394,39 @@ public BranchedKStreamX split(final Named named) { } @Override - public ImprovedKStream merge(final KStream stream) { + public KStreamX merge(final KStream stream) { final KStream other = StreamsContext.maybeUnwrap(stream); return this.context.wrap(this.wrapped.merge(other)); } @Override - public ImprovedKStream merge(final KStream stream, final Named named) { + public KStreamX merge(final KStream stream, final Named named) { final KStream other = StreamsContext.maybeUnwrap(stream); return this.context.wrap(this.wrapped.merge(other, named)); } @Override - public ImprovedKStream through(final String topic) { + public KStreamX through(final String topic) { return this.context.wrap(this.wrapped.through(topic)); } @Override - public ImprovedKStream through(final String topic, final Produced produced) { + public KStreamX through(final String topic, final Produced produced) { return this.context.wrap(this.wrapped.through(topic, produced)); } @Override - public ImprovedKStream repartition() { + public KStreamX repartition() { return this.context.wrap(this.wrapped.repartition()); } @Override - public ImprovedKStream repartition(final Repartitioned repartitioned) { + public KStreamX repartition(final Repartitioned repartitioned) { return this.context.wrap(this.wrapped.repartition(repartitioned)); } @Override - public ImprovedKStream repartition(final AutoRepartitioned repartitioned) { + public KStreamX repartition(final AutoRepartitioned repartitioned) { return this.repartition(repartitioned.configure(this.context.getConfigurator())); } @@ -570,14 +570,14 @@ public KGroupedStreamX groupByKey(final AutoGrouped grouped) { } @Override - public ImprovedKStream join(final KStream otherStream, + public KStreamX join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { final KStream other = StreamsContext.maybeUnwrap(otherStream); return this.context.wrap(this.wrapped.join(other, joiner, windows)); } @Override - public ImprovedKStream join(final KStream otherStream, + public KStreamX join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -585,7 +585,7 @@ public ImprovedKStream join(final KStream otherStream, } @Override - public ImprovedKStream join(final KStream otherStream, + public KStreamX join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -593,14 +593,14 @@ public ImprovedKStream join(final KStream otherStream, } @Override - public ImprovedKStream join(final KStream otherStream, + public KStreamX join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final AutoStreamJoined streamJoined) { return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream join(final KStream otherStream, + public KStreamX join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -608,21 +608,21 @@ public ImprovedKStream join(final KStream otherStream, } @Override - public ImprovedKStream join(final KStream otherStream, + public KStreamX join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final AutoStreamJoined streamJoined) { return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream leftJoin(final KStream otherStream, + public KStreamX leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { final KStream other = StreamsContext.maybeUnwrap(otherStream); return this.context.wrap(this.wrapped.leftJoin(other, joiner, windows)); } @Override - public ImprovedKStream leftJoin(final KStream otherStream, + public KStreamX leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -630,7 +630,7 @@ public ImprovedKStream leftJoin(final KStream otherStream } @Override - public ImprovedKStream leftJoin(final KStream otherStream, + public KStreamX leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -638,14 +638,14 @@ public ImprovedKStream leftJoin(final KStream otherStream } @Override - public ImprovedKStream leftJoin(final KStream otherStream, + public KStreamX leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final AutoStreamJoined streamJoined) { return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream leftJoin(final KStream otherStream, + public KStreamX leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -653,21 +653,21 @@ public ImprovedKStream leftJoin(final KStream otherStream } @Override - public ImprovedKStream leftJoin(final KStream otherStream, + public KStreamX leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final AutoStreamJoined streamJoined) { return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream outerJoin(final KStream otherStream, + public KStreamX outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows) { final KStream other = StreamsContext.maybeUnwrap(otherStream); return this.context.wrap(this.wrapped.outerJoin(other, joiner, windows)); } @Override - public ImprovedKStream outerJoin(final KStream otherStream, + public KStreamX outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -675,7 +675,7 @@ public ImprovedKStream outerJoin(final KStream otherStrea } @Override - public ImprovedKStream outerJoin(final KStream otherStream, + public KStreamX outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final StreamJoined streamJoined) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -683,14 +683,14 @@ public ImprovedKStream outerJoin(final KStream otherStrea } @Override - public ImprovedKStream outerJoin(final KStream otherStream, + public KStreamX outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, final AutoStreamJoined streamJoined) { return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream outerJoin(final KStream otherStream, + public KStreamX outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final StreamJoined streamJoined) { final KStream other = StreamsContext.maybeUnwrap(otherStream); @@ -698,41 +698,41 @@ public ImprovedKStream outerJoin(final KStream otherStrea } @Override - public ImprovedKStream outerJoin(final KStream otherStream, + public KStreamX outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, final AutoStreamJoined streamJoined) { return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream join(final KTable table, + public KStreamX join(final KTable table, final ValueJoiner joiner) { final KTable other = StreamsContext.maybeUnwrap(table); return this.context.wrap(this.wrapped.join(other, joiner)); } @Override - public ImprovedKStream join(final KTable table, + public KStreamX join(final KTable table, final ValueJoinerWithKey joiner) { final KTable other = StreamsContext.maybeUnwrap(table); return this.context.wrap(this.wrapped.join(other, joiner)); } @Override - public ImprovedKStream join(final KTable table, + public KStreamX join(final KTable table, final ValueJoiner joiner, final Joined joined) { final KTable other = StreamsContext.maybeUnwrap(table); return this.context.wrap(this.wrapped.join(other, joiner, joined)); } @Override - public ImprovedKStream join(final KTable table, + public KStreamX join(final KTable table, final ValueJoiner joiner, final AutoJoined joined) { return this.join(table, joiner, joined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream join(final KTable table, + public KStreamX join(final KTable table, final ValueJoinerWithKey joiner, final Joined joined) { final KTable other = StreamsContext.maybeUnwrap(table); @@ -740,41 +740,41 @@ public ImprovedKStream join(final KTable table, } @Override - public ImprovedKStream join(final KTable table, + public KStreamX join(final KTable table, final ValueJoinerWithKey joiner, final AutoJoined joined) { return this.join(table, joiner, joined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream leftJoin(final KTable table, + public KStreamX leftJoin(final KTable table, final ValueJoiner joiner) { final KTable other = StreamsContext.maybeUnwrap(table); return this.context.wrap(this.wrapped.leftJoin(other, joiner)); } @Override - public ImprovedKStream leftJoin(final KTable table, + public KStreamX leftJoin(final KTable table, final ValueJoinerWithKey joiner) { final KTable other = StreamsContext.maybeUnwrap(table); return this.context.wrap(this.wrapped.leftJoin(other, joiner)); } @Override - public ImprovedKStream leftJoin(final KTable table, + public KStreamX leftJoin(final KTable table, final ValueJoiner joiner, final Joined joined) { final KTable other = StreamsContext.maybeUnwrap(table); return this.context.wrap(this.wrapped.leftJoin(other, joiner, joined)); } @Override - public ImprovedKStream leftJoin(final KTable table, + public KStreamX leftJoin(final KTable table, final ValueJoiner joiner, final AutoJoined joined) { return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream leftJoin(final KTable table, + public KStreamX leftJoin(final KTable table, final ValueJoinerWithKey joiner, final Joined joined) { final KTable other = StreamsContext.maybeUnwrap(table); @@ -782,91 +782,91 @@ public ImprovedKStream leftJoin(final KTable table, } @Override - public ImprovedKStream leftJoin(final KTable table, + public KStreamX leftJoin(final KTable table, final ValueJoinerWithKey joiner, final AutoJoined joined) { return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); } @Override - public ImprovedKStream join(final GlobalKTable globalTable, + public KStreamX join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner joiner) { return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner)); } @Override - public ImprovedKStream join(final GlobalKTable globalTable, + public KStreamX join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey joiner) { return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner)); } @Override - public ImprovedKStream join(final GlobalKTable globalTable, + public KStreamX join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner joiner, final Named named) { return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner, named)); } @Override - public ImprovedKStream join(final GlobalKTable globalTable, + public KStreamX join(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey joiner, final Named named) { return this.context.wrap(this.wrapped.join(globalTable, keySelector, joiner, named)); } @Override - public ImprovedKStream leftJoin(final GlobalKTable globalTable, + public KStreamX leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner valueJoiner) { return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); } @Override - public ImprovedKStream leftJoin(final GlobalKTable globalTable, + public KStreamX leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey valueJoiner) { return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner)); } @Override - public ImprovedKStream leftJoin(final GlobalKTable globalTable, + public KStreamX leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoiner valueJoiner, final Named named) { return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); } @Override - public ImprovedKStream leftJoin(final GlobalKTable globalTable, + public KStreamX leftJoin(final GlobalKTable globalTable, final KeyValueMapper keySelector, final ValueJoinerWithKey valueJoiner, final Named named) { return this.context.wrap(this.wrapped.leftJoin(globalTable, keySelector, valueJoiner, named)); } @Override - public ImprovedKStream transform( + public KStreamX transform( final TransformerSupplier> transformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transform(transformerSupplier, stateStoreNames)); } @Override - public ImprovedKStream transform( + public KStreamX transform( final TransformerSupplier> transformerSupplier, final Named named, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transform(transformerSupplier, named, stateStoreNames)); } @Override - public ImprovedKStream flatTransform( + public KStreamX flatTransform( final TransformerSupplier>> transformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.flatTransform(transformerSupplier, stateStoreNames)); } @Override - public ImprovedKStream flatTransform( + public KStreamX flatTransform( final TransformerSupplier>> transformerSupplier, final Named named, final String... stateStoreNames) { @@ -874,28 +874,28 @@ public ImprovedKStream flatTransform( } @Override - public ImprovedKStream transformValues( + public KStreamX transformValues( final ValueTransformerSupplier valueTransformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); } @Override - public ImprovedKStream transformValues( + public KStreamX transformValues( final ValueTransformerSupplier valueTransformerSupplier, final Named named, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, named, stateStoreNames)); } @Override - public ImprovedKStream transformValues( + public KStreamX transformValues( final ValueTransformerWithKeySupplier valueTransformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.transformValues(valueTransformerSupplier, stateStoreNames)); } @Override - public ImprovedKStream transformValues( + public KStreamX transformValues( final ValueTransformerWithKeySupplier valueTransformerSupplier, final Named named, final String... stateStoreNames) { @@ -903,14 +903,14 @@ public ImprovedKStream transformValues( } @Override - public ImprovedKStream flatTransformValues( + public KStreamX flatTransformValues( final ValueTransformerSupplier> valueTransformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); } @Override - public ImprovedKStream flatTransformValues( + public KStreamX flatTransformValues( final ValueTransformerSupplier> valueTransformerSupplier, final Named named, final String... stateStoreNames) { return this.context.wrap( @@ -918,14 +918,14 @@ public ImprovedKStream flatTransformValues( } @Override - public ImprovedKStream flatTransformValues( + public KStreamX flatTransformValues( final ValueTransformerWithKeySupplier> valueTransformerSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.flatTransformValues(valueTransformerSupplier, stateStoreNames)); } @Override - public ImprovedKStream flatTransformValues( + public KStreamX flatTransformValues( final ValueTransformerWithKeySupplier> valueTransformerSupplier, final Named named, final String... stateStoreNames) { @@ -948,7 +948,7 @@ public void process( } @Override - public ImprovedKStream process( + public KStreamX process( final ProcessorSupplier processorSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.process(processorSupplier, stateStoreNames)); @@ -973,7 +973,7 @@ public KErrorStream processCapturingErrors( } @Override - public ImprovedKStream process( + public KStreamX process( final ProcessorSupplier processorSupplier, final Named named, final String... stateStoreNames) { return this.context.wrap(this.wrapped.process(processorSupplier, named, stateStoreNames)); @@ -998,7 +998,7 @@ public KErrorStream processCapturingErrors( } @Override - public ImprovedKStream processValues( + public KStreamX processValues( final FixedKeyProcessorSupplier processorSupplier, final String... stateStoreNames) { return this.context.wrap(this.wrapped.processValues(processorSupplier, stateStoreNames)); @@ -1023,7 +1023,7 @@ public KErrorStream processValuesCapturingErrors( } @Override - public ImprovedKStream processValues( + public KStreamX processValues( final FixedKeyProcessorSupplier processorSupplier, final Named named, final String... stateStoreNames) { return this.context.wrap(this.wrapped.processValues(processorSupplier, named, stateStoreNames)); @@ -1049,81 +1049,81 @@ public KErrorStream processValuesCapturingErrors( private KeyValueKErrorStream mapCapturingErrorsInternal( final KeyValueMapper>> mapper) { - final ImprovedKStream> map = this.map(mapper); + final KStreamX> map = this.map(mapper); return new KeyValueKErrorStream<>(map); } private KeyValueKErrorStream mapCapturingErrorsInternal( final KeyValueMapper>> mapper, final Named named) { - final ImprovedKStream> map = this.map(mapper, named); + final KStreamX> map = this.map(mapper, named); return new KeyValueKErrorStream<>(map); } private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapper> mapper) { - final ImprovedKStream> map = this.mapValues(mapper); + final KStreamX> map = this.mapValues(mapper); return new ValueKErrorStream<>(map); } private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapper> mapper, final Named named) { - final ImprovedKStream> map = this.mapValues(mapper, named); + final KStreamX> map = this.mapValues(mapper, named); return new ValueKErrorStream<>(map); } private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapperWithKey> mapper) { - final ImprovedKStream> map = this.mapValues(mapper); + final KStreamX> map = this.mapValues(mapper); return new ValueKErrorStream<>(map); } private ValueKErrorStream mapValuesCapturingErrorsInternal( final ValueMapperWithKey> mapper, final Named named) { - final ImprovedKStream> map = this.mapValues(mapper, named); + final KStreamX> map = this.mapValues(mapper, named); return new ValueKErrorStream<>(map); } private KeyValueKErrorStream flatMapCapturingErrorsInternal( final KeyValueMapper>>> mapper) { - final ImprovedKStream> map = this.flatMap(mapper); + final KStreamX> map = this.flatMap(mapper); return new KeyValueKErrorStream<>(map); } private KeyValueKErrorStream flatMapCapturingErrorsInternal( final KeyValueMapper>>> mapper, final Named named) { - final ImprovedKStream> map = this.flatMap(mapper, named); + final KStreamX> map = this.flatMap(mapper, named); return new KeyValueKErrorStream<>(map); } private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapper>> mapper) { - final ImprovedKStream> map = this.flatMapValues(mapper); + final KStreamX> map = this.flatMapValues(mapper); return new ValueKErrorStream<>(map); } private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapper>> mapper, final Named named) { - final ImprovedKStream> map = this.flatMapValues(mapper, named); + final KStreamX> map = this.flatMapValues(mapper, named); return new ValueKErrorStream<>(map); } private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapperWithKey>> mapper) { - final ImprovedKStream> map = this.flatMapValues(mapper); + final KStreamX> map = this.flatMapValues(mapper); return new ValueKErrorStream<>(map); } private ValueKErrorStream flatMapValuesCapturingErrorsInternal( final ValueMapperWithKey>> mapper, final Named named) { - final ImprovedKStream> map = this.flatMapValues(mapper, named); + final KStreamX> map = this.flatMapValues(mapper, named); return new ValueKErrorStream<>(map); } private KeyValueKErrorStream processCapturingErrorsInternal( final ProcessorSupplier> processorSupplier, final String... stateStoreNames) { - final ImprovedKStream> map = + final KStreamX> map = this.process(processorSupplier, stateStoreNames); return new KeyValueKErrorStream<>(map); } @@ -1131,7 +1131,7 @@ private KeyValueKErrorStream processCapturingErro private KeyValueKErrorStream processCapturingErrorsInternal( final ProcessorSupplier> processorSupplier, final Named named, final String... stateStoreNames) { - final ImprovedKStream> map = + final KStreamX> map = this.process(processorSupplier, named, stateStoreNames); return new KeyValueKErrorStream<>(map); } @@ -1139,14 +1139,14 @@ private KeyValueKErrorStream processCapturingErro private ValueKErrorStream processValuesCapturingErrorsInternal( final FixedKeyProcessorSupplier> processorSupplier, final String... stateStoreNames) { - final ImprovedKStream> map = this.processValues(processorSupplier, stateStoreNames); + final KStreamX> map = this.processValues(processorSupplier, stateStoreNames); return new ValueKErrorStream<>(map); } private ValueKErrorStream processValuesCapturingErrorsInternal( final FixedKeyProcessorSupplier> processorSupplier, final Named named, final String... stateStoreNames) { - final ImprovedKStream> map = + final KStreamX> map = this.processValues(processorSupplier, named, stateStoreNames); return new ValueKErrorStream<>(map); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index af6773cd..b5a3fa84 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -154,16 +154,16 @@ KTableX mapValues(ValueMapperWithKey> materialized); @Override - ImprovedKStream toStream(); + KStreamX toStream(); @Override - ImprovedKStream toStream(Named named); + KStreamX toStream(Named named); @Override - ImprovedKStream toStream(KeyValueMapper mapper); + KStreamX toStream(KeyValueMapper mapper); @Override - ImprovedKStream toStream(KeyValueMapper mapper, Named named); + KStreamX toStream(KeyValueMapper mapper, Named named); @Override KTableX suppress(Suppressed suppressed); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java index 2a773888..7063ed86 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java @@ -190,22 +190,22 @@ public KTableX mapValues(final ValueMapperWithKey toStream() { + public KStreamX toStream() { return this.context.wrap(this.wrapped.toStream()); } @Override - public ImprovedKStream toStream(final Named named) { + public KStreamX toStream(final Named named) { return this.context.wrap(this.wrapped.toStream(named)); } @Override - public ImprovedKStream toStream(final KeyValueMapper mapper) { + public KStreamX toStream(final KeyValueMapper mapper) { return this.context.wrap(this.wrapped.toStream(mapper)); } @Override - public ImprovedKStream toStream(final KeyValueMapper mapper, + public KStreamX toStream(final KeyValueMapper mapper, final Named named) { return this.context.wrap(this.wrapped.toStream(mapper, named)); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java index 7b2e59fb..4f4f968a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KeyValueKErrorStream.java @@ -30,25 +30,25 @@ @RequiredArgsConstructor class KeyValueKErrorStream implements KErrorStream { - private final @NonNull ImprovedKStream> stream; + private final @NonNull KStreamX> stream; @Override - public ImprovedKStream values() { + public KStreamX values() { return this.stream.flatMapValues(ProcessedKeyValue::getValues); } @Override - public ImprovedKStream values(final Named named) { + public KStreamX values(final Named named) { return this.stream.flatMapValues(ProcessedKeyValue::getValues, named); } @Override - public ImprovedKStream> errors() { + public KStreamX> errors() { return this.stream.flatMap(ProcessedKeyValue::getErrors); } @Override - public ImprovedKStream> errors(final Named named) { + public KStreamX> errors(final Named named) { return this.stream.flatMap(ProcessedKeyValue::getErrors, named); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java index 73382fba..2d33f90b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamsContext.java @@ -53,9 +53,9 @@ public class StreamsContext { Configurator configurator; static KStream maybeUnwrap(final KStream stream) { - if (stream instanceof ImprovedKStreamImpl) { + if (stream instanceof KStreamXImpl) { // Kafka Streams internally casts KStream to KStreamImpl in some cases - return ((ImprovedKStreamImpl) stream).getWrapped(); + return ((KStreamXImpl) stream).getWrapped(); } return stream; } @@ -79,15 +79,15 @@ static KTable maybeUnwrap(final KTable table) { /** * Wrap a {@link KStream} and add methods to simplify Serde configuration, error handling, and topic access * @param stream stream to be wrapped - * @return {@link ImprovedKStream} + * @return {@link KStreamX} * @param type of keys in the stream * @param type of values in the stream */ - public ImprovedKStream wrap(final KStream stream) { - if (stream instanceof ImprovedKStream) { - return (ImprovedKStream) stream; + public KStreamX wrap(final KStream stream) { + if (stream instanceof KStreamX) { + return (KStreamX) stream; } - return new ImprovedKStreamImpl<>(stream, this); + return new KStreamXImpl<>(stream, this); } /** diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index ea0405ec..f97982f8 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -52,42 +52,42 @@ public class TopologyBuilder { /** * @see StreamsBuilder#stream(String) */ - public ImprovedKStream stream(final String topic) { + public KStreamX stream(final String topic) { return this.getContext().wrap(this.streamsBuilder.stream(topic)); } /** * @see StreamsBuilder#stream(String, Consumed) */ - public ImprovedKStream stream(final String topic, final Consumed consumed) { + public KStreamX stream(final String topic, final Consumed consumed) { return this.getContext().wrap(this.streamsBuilder.stream(topic, consumed)); } /** * @see StreamsBuilder#stream(String, Consumed) */ - public ImprovedKStream stream(final String topic, final AutoConsumed consumed) { + public KStreamX stream(final String topic, final AutoConsumed consumed) { return this.stream(topic, consumed.configure(this.createConfigurator())); } /** * @see StreamsBuilder#stream(Collection) */ - public ImprovedKStream stream(final Collection topics) { + public KStreamX stream(final Collection topics) { return this.getContext().wrap(this.streamsBuilder.stream(topics)); } /** * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream stream(final Collection topics, final Consumed consumed) { + public KStreamX stream(final Collection topics, final Consumed consumed) { return this.getContext().wrap(this.streamsBuilder.stream(topics, consumed)); } /** * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream stream(final Collection topics, + public KStreamX stream(final Collection topics, final AutoConsumed consumed) { return this.stream(topics, consumed.configure(this.createConfigurator())); } @@ -95,21 +95,21 @@ public ImprovedKStream stream(final Collection topics, /** * @see StreamsBuilder#stream(Pattern) */ - public ImprovedKStream stream(final Pattern topicPattern) { + public KStreamX stream(final Pattern topicPattern) { return this.getContext().wrap(this.streamsBuilder.stream(topicPattern)); } /** * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream stream(final Pattern topicPattern, final Consumed consumed) { + public KStreamX stream(final Pattern topicPattern, final Consumed consumed) { return this.getContext().wrap(this.streamsBuilder.stream(topicPattern, consumed)); } /** * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream stream(final Pattern topicPattern, final AutoConsumed consumed) { + public KStreamX stream(final Pattern topicPattern, final AutoConsumed consumed) { return this.stream(topicPattern, consumed.configure(this.createConfigurator())); } @@ -121,7 +121,7 @@ public ImprovedKStream stream(final Pattern topicPattern, final Aut * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream streamInput(final Consumed consumed) { + public KStreamX streamInput(final Consumed consumed) { return this.stream(this.topics.getInputTopics(), consumed); } @@ -133,7 +133,7 @@ public ImprovedKStream streamInput(final Consumed consumed) { * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream streamInput(final AutoConsumed consumed) { + public KStreamX streamInput(final AutoConsumed consumed) { return this.streamInput(consumed.configure(this.createConfigurator())); } @@ -144,7 +144,7 @@ public ImprovedKStream streamInput(final AutoConsumed consume * @param type of values * @see StreamsBuilder#stream(Collection) */ - public ImprovedKStream streamInput() { + public KStreamX streamInput() { return this.stream(this.topics.getInputTopics()); } @@ -157,7 +157,7 @@ public ImprovedKStream streamInput() { * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream streamInput(final String label, final Consumed consumed) { + public KStreamX streamInput(final String label, final Consumed consumed) { return this.stream(this.topics.getInputTopics(label), consumed); } @@ -170,7 +170,7 @@ public ImprovedKStream streamInput(final String label, final Consum * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public ImprovedKStream streamInput(final String label, final AutoConsumed consumed) { + public KStreamX streamInput(final String label, final AutoConsumed consumed) { return this.streamInput(label, consumed.configure(this.createConfigurator())); } @@ -182,7 +182,7 @@ public ImprovedKStream streamInput(final String label, final AutoCo * @param type of values * @see StreamsBuilder#stream(Collection) */ - public ImprovedKStream streamInput(final String label) { + public KStreamX streamInput(final String label) { return this.stream(this.topics.getInputTopics(label)); } @@ -194,7 +194,7 @@ public ImprovedKStream streamInput(final String label) { * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream streamInputPattern(final Consumed consumed) { + public KStreamX streamInputPattern(final Consumed consumed) { return this.stream(this.topics.getInputPattern(), consumed); } @@ -206,7 +206,7 @@ public ImprovedKStream streamInputPattern(final Consumed cons * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream streamInputPattern(final AutoConsumed consumed) { + public KStreamX streamInputPattern(final AutoConsumed consumed) { return this.streamInputPattern(consumed.configure(this.createConfigurator())); } @@ -217,7 +217,7 @@ public ImprovedKStream streamInputPattern(final AutoConsumed * @param type of values * @see StreamsBuilder#stream(Pattern) */ - public ImprovedKStream streamInputPattern() { + public KStreamX streamInputPattern() { return this.stream(this.topics.getInputPattern()); } @@ -230,7 +230,7 @@ public ImprovedKStream streamInputPattern() { * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream streamInputPattern(final String label, final Consumed consumed) { + public KStreamX streamInputPattern(final String label, final Consumed consumed) { return this.stream(this.topics.getInputPattern(label), consumed); } @@ -243,7 +243,7 @@ public ImprovedKStream streamInputPattern(final String label, final * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ - public ImprovedKStream streamInputPattern(final String label, + public KStreamX streamInputPattern(final String label, final AutoConsumed consumed) { return this.streamInputPattern(label, consumed.configure(this.createConfigurator())); } @@ -256,7 +256,7 @@ public ImprovedKStream streamInputPattern(final String label, * @param type of values * @see StreamsBuilder#stream(Pattern) */ - public ImprovedKStream streamInputPattern(final String label) { + public KStreamX streamInputPattern(final String label) { return this.stream(this.topics.getInputPattern(label)); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java index eb6681ff..8442edab 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ValueKErrorStream.java @@ -30,25 +30,25 @@ @RequiredArgsConstructor class ValueKErrorStream implements KErrorStream { - private final @NonNull ImprovedKStream> stream; + private final @NonNull KStreamX> stream; @Override - public ImprovedKStream values() { + public KStreamX values() { return this.stream.flatMapValues(ProcessedValue::getValues); } @Override - public ImprovedKStream values(final Named named) { + public KStreamX values(final Named named) { return this.stream.flatMapValues(ProcessedValue::getValues, named); } @Override - public ImprovedKStream> errors() { + public KStreamX> errors() { return this.stream.flatMapValues(ProcessedValue::getErrors); } @Override - public ImprovedKStream> errors(final Named named) { + public KStreamX> errors(final Named named) { return this.stream.flatMapValues(ProcessedValue::getErrors, named); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index dc323b58..fb73e72a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -24,7 +24,7 @@ package com.bakdata.kafka; -import static com.bakdata.kafka.ImprovedKStreamTest.startApp; +import static com.bakdata.kafka.KStreamXTest.startApp; import com.bakdata.fluent_kafka_streams_tests.TestTopology; import org.apache.kafka.common.serialization.Serdes; @@ -37,7 +37,7 @@ void shouldReduce() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KGroupedStreamX grouped = input.groupByKey(); final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2); reduced.toStream().to("output"); @@ -64,7 +64,7 @@ void shouldReduceUsingMaterialized() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final KGroupedStreamX grouped = input.groupByKey( @@ -102,7 +102,7 @@ void shouldAggregate() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KGroupedStreamX grouped = input.groupByKey(); final KTableX aggregated = grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value); @@ -130,7 +130,7 @@ void shouldAggregateUsingMaterialized() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final KGroupedStreamX grouped = input.groupByKey( diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java similarity index 90% rename from streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java rename to streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 162d2918..7cc8bca6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ImprovedKStreamTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -56,7 +56,7 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.STRICT_STUBS) -class ImprovedKStreamTest { +class KStreamXTest { static TestTopology startApp(final StreamsApp app, final StreamsTopicConfig topicConfig) { final ConfiguredStreamsApp configuredStreamsApp = @@ -72,7 +72,7 @@ void shouldWriteToOutput() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.toOutputTopic(); } }; @@ -94,7 +94,7 @@ void shouldWriteToOutputUsingProduced() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.toOutputTopic(AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -124,7 +124,7 @@ void shouldWriteToLabeledOutput() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.toOutputTopic("label"); } }; @@ -146,7 +146,7 @@ void shouldWriteToLabeledOutputUsingProduced() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.toOutputTopic("label", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -176,7 +176,7 @@ void shouldWriteToError() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.toErrorTopic(); } }; @@ -198,7 +198,7 @@ void shouldWriteToErrorUsingProduced() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.toErrorTopic(AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -230,7 +230,7 @@ void shouldMap() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.map(mapper).to("output"); } }; @@ -252,7 +252,7 @@ void shouldMapValues() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.mapValues(mapper).to("output"); } }; @@ -274,7 +274,7 @@ void shouldMapValuesWithKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.mapValues(mapper).to("output"); } }; @@ -296,7 +296,7 @@ void shouldFlatMap() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.flatMap(mapper).to("output"); } }; @@ -318,7 +318,7 @@ void shouldFlatMapValues() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.flatMapValues(mapper).to("output"); } }; @@ -340,7 +340,7 @@ void shouldFlatMapValuesWithKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.flatMapValues(mapper).to("output"); } }; @@ -371,8 +371,8 @@ public void process(final Record inputRecord) { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream processed = input.process(processor); + final KStreamX input = builder.stream("input"); + final KStreamX processed = input.process(processor); processed.to("output"); } }; @@ -403,8 +403,8 @@ public void process(final FixedKeyRecord inputRecord) { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream processed = input.processValues(processor); + final KStreamX input = builder.stream("input"); + final KStreamX processed = input.processValues(processor); processed.to("output"); } }; @@ -427,7 +427,7 @@ void shouldFilter() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.filter(predicate).to("output"); } }; @@ -452,7 +452,7 @@ void shouldFilterNot() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.filterNot(predicate).to("output"); } }; @@ -474,7 +474,7 @@ void shouldSelectKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); input.selectKey((k, v) -> v).to("output"); } }; @@ -498,7 +498,7 @@ void shouldMapCapturingErrors() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.mapCapturingErrors(mapper); processed.values().to("output"); @@ -536,7 +536,7 @@ void shouldMapValuesCapturingErrors() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.mapValuesCapturingErrors(mapper); processed.values().to("output"); @@ -574,7 +574,7 @@ void shouldMapValuesWithKeyCapturingErrors() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.mapValuesCapturingErrors(mapper); processed.values().to("output"); @@ -612,7 +612,7 @@ void shouldFlatMapCapturingErrors() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.flatMapCapturingErrors(mapper); processed.values().to("output"); @@ -650,7 +650,7 @@ void shouldFlatMapValuesCapturingErrors() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.flatMapValuesCapturingErrors(mapper); processed.values().to("output"); @@ -688,7 +688,7 @@ void shouldFlatMapValuesWithKeyCapturingErrors() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.flatMapValuesCapturingErrors(mapper); processed.values().to("output"); @@ -737,7 +737,7 @@ public void process(final Record inputRecord) { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.processCapturingErrors(processor); processed.values().to("output"); @@ -786,7 +786,7 @@ public void process(final FixedKeyRecord inputRecord) { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KErrorStream processed = input.processValuesCapturingErrors(processor); processed.values().to("output"); @@ -821,8 +821,8 @@ void shouldRepartition() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream repartitioned = input.repartition(); + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = input.repartition(); repartitioned.to("output"); } }; @@ -843,10 +843,10 @@ void shouldRepartitionUsingRepartitioned() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream repartitioned = input.repartition( + final KStreamX repartitioned = input.repartition( AutoRepartitioned.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); repartitioned.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -874,7 +874,7 @@ void shouldConvertToTable() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KTableX table = input.toTable(); table.toStream().to("output"); } @@ -896,7 +896,7 @@ void shouldConvertToTableUsingMaterialized() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final KTableX table = input.toTable( @@ -927,9 +927,9 @@ void shouldJoin() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream otherInput = builder.stream("other_input"); - final ImprovedKStream joined = input.join(otherInput, + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.join(otherInput, (v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); @@ -954,9 +954,9 @@ void shouldJoinWithKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream otherInput = builder.stream("other_input"); - final ImprovedKStream joined = input.join(otherInput, + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.join(otherInput, (k, v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); @@ -981,9 +981,9 @@ void shouldLeftJoin() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream otherInput = builder.stream("other_input"); - final ImprovedKStream joined = input.leftJoin(otherInput, + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); @@ -1015,9 +1015,9 @@ void shouldLeftJoinWithKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream otherInput = builder.stream("other_input"); - final ImprovedKStream joined = input.leftJoin(otherInput, + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); @@ -1049,9 +1049,9 @@ void shouldOuterJoin() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream otherInput = builder.stream("other_input"); - final ImprovedKStream joined = input.outerJoin(otherInput, + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); @@ -1083,9 +1083,9 @@ void shouldOuterJoinWithKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); - final ImprovedKStream otherInput = builder.stream("other_input"); - final ImprovedKStream joined = input.outerJoin(otherInput, + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.outerJoin(otherInput, (k, v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); @@ -1117,13 +1117,13 @@ void shouldJoinUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream otherInput = builder.stream("other_input", + final KStreamX otherInput = builder.stream("other_input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream joined = input.join(otherInput, + final KStreamX joined = input.join(otherInput, Long::sum, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), @@ -1157,13 +1157,13 @@ void shouldJoinWithKeyUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream otherInput = builder.stream("other_input", + final KStreamX otherInput = builder.stream("other_input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream joined = input.join(otherInput, + final KStreamX joined = input.join(otherInput, (k, v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), @@ -1197,13 +1197,13 @@ void shouldLeftJoinUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream otherInput = builder.stream("other_input", + final KStreamX otherInput = builder.stream("other_input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream joined = input.leftJoin(otherInput, + final KStreamX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), @@ -1246,13 +1246,13 @@ void shouldLeftJoinWithKeyUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream otherInput = builder.stream("other_input", + final KStreamX otherInput = builder.stream("other_input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream joined = input.leftJoin(otherInput, + final KStreamX joined = input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), @@ -1295,13 +1295,13 @@ void shouldOuterJoinUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream otherInput = builder.stream("other_input", + final KStreamX otherInput = builder.stream("other_input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream joined = input.outerJoin(otherInput, + final KStreamX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), @@ -1344,13 +1344,13 @@ void shouldOuterJoinWithKeyUsingJoined() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream otherInput = builder.stream("other_input", + final KStreamX otherInput = builder.stream("other_input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - final ImprovedKStream joined = input.outerJoin(otherInput, + final KStreamX joined = input.outerJoin(otherInput, (k, v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), @@ -1393,7 +1393,7 @@ void shouldGroupByKey() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KGroupedStreamX grouped = input.groupByKey(); final KTableX count = grouped.count(); count.toStream().to("output"); @@ -1421,7 +1421,7 @@ void shouldGroupByKeyUsingGrouped() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final KGroupedStreamX grouped = input.groupByKey( @@ -1457,7 +1457,7 @@ void shouldGroupBy() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input"); + final KStreamX input = builder.stream("input"); final KGroupedStreamX grouped = input.groupBy((k, v) -> v); final KTableX count = grouped.count(); count.toStream().to("output"); @@ -1485,7 +1485,7 @@ void shouldGroupByUsingGrouped() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.stream("input", + final KStreamX input = builder.stream("input", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); final KGroupedStreamX grouped = input.groupBy((k, v) -> v, diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index a4eb2662..b01d4705 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -24,7 +24,7 @@ package com.bakdata.kafka; -import static com.bakdata.kafka.ImprovedKStreamTest.startApp; +import static com.bakdata.kafka.KStreamXTest.startApp; import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.util.List; @@ -40,7 +40,7 @@ void shouldReadFromInput() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.to("output"); } }; @@ -62,7 +62,7 @@ void shouldReadFromInputUsingConsumed() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput( + final KStreamX input = builder.streamInput( AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -92,7 +92,7 @@ void shouldReadFromLabeledInput() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput("label"); + final KStreamX input = builder.streamInput("label"); input.to("output"); } }; @@ -114,7 +114,7 @@ void shouldReadFromLabeledInputUsingConsumed() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput("label", + final KStreamX input = builder.streamInput("label", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -144,7 +144,7 @@ void shouldReadFromPatternInput() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInputPattern(); + final KStreamX input = builder.streamInputPattern(); input.to("output"); } }; @@ -166,7 +166,7 @@ void shouldReadFromPatternInputUsingConsumed() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInputPattern( + final KStreamX input = builder.streamInputPattern( AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), @@ -196,7 +196,7 @@ void shouldReadFromLabeledPatternInput() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInputPattern("label"); + final KStreamX input = builder.streamInputPattern("label"); input.to("output"); } }; @@ -218,7 +218,7 @@ void shouldReadFromLabeledPatternInputUsingConsumed() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInputPattern("label", + final KStreamX input = builder.streamInputPattern("label", AutoConsumed.with(Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java index 0a4b8198..8c6b919d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java @@ -32,7 +32,7 @@ import com.bakdata.kafka.AppConfiguration; import com.bakdata.kafka.ConfiguredStreamsApp; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.KafkaTest; import com.bakdata.kafka.KafkaTestClient; import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; @@ -243,7 +243,7 @@ private static class ErrorApplication implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.map((k, v) -> {throw new RuntimeException("Error in map");}) .toOutputTopic(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java index c3e1c227..5a115f2c 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/ComplexTopologyApplication.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -47,10 +47,10 @@ public class ComplexTopologyApplication implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.to(THROUGH_TOPIC); - final ImprovedKStream through = builder.stream(THROUGH_TOPIC); + final KStreamX through = builder.stream(THROUGH_TOPIC); final KTableX, TestRecord> reduce = through .groupByKey() .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMillis(5L))) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java index eee7c32d..6e105416 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/LabeledInputTopics.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -36,7 +36,7 @@ public class LabeledInputTopics implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput("label"); + final KStreamX input = builder.streamInput("label"); input.toOutputTopic(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java index 8833fae6..846b62a8 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/Mirror.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -36,7 +36,7 @@ public class Mirror implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.toOutputTopic(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java index b2cce89b..012a1d4f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorKeyWithAvro.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -38,7 +38,7 @@ public class MirrorKeyWithAvro implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.toOutputTopic(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java index 656c8f4d..f86c0517 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorValueWithAvro.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -38,7 +38,7 @@ public class MirrorValueWithAvro implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream input = builder.streamInput(); + final KStreamX input = builder.streamInput(); input.toOutputTopic(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java index 890ae563..30dd0130 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java @@ -26,7 +26,7 @@ import com.bakdata.kafka.AutoConsumed; import com.bakdata.kafka.AutoProduced; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.Preconfigured; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -53,7 +53,7 @@ public static Preconfigured> newValueSerde() { public void buildTopology(final TopologyBuilder builder) { final Preconfigured> valueSerde = newValueSerde(); final Preconfigured> keySerde = newKeySerde(); - final ImprovedKStream input = + final KStreamX input = builder.streamInput(AutoConsumed.with(keySerde, valueSerde)); input.toOutputTopic(AutoProduced.with(keySerde, valueSerde)); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java index e4f82f20..c4047d96 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCount.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -43,7 +43,7 @@ public class WordCount implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream textLines = builder.streamInput(); + final KStreamX textLines = builder.streamInput(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); final KTableX wordCounts = textLines diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java index c9d35d04..4d8f5afd 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/WordCountPattern.java @@ -24,7 +24,7 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.ImprovedKStream; +import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.KTableX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; @@ -44,7 +44,7 @@ public class WordCountPattern implements StreamsApp { @Override public void buildTopology(final TopologyBuilder builder) { - final ImprovedKStream textLines = builder.streamInputPattern(); + final KStreamX textLines = builder.streamInputPattern(); final Pattern pattern = Pattern.compile("\\W+", Pattern.UNICODE_CHARACTER_CLASS); final KTableX wordCounts = textLines From d53721a3bc9b415fa201e89ae7ae4d953c9e8794 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 18:35:22 +0100 Subject: [PATCH 32/72] Add docs --- .../src/main/java/com/bakdata/kafka/KStreamX.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java index c44c2a3b..4deac9f6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java @@ -899,6 +899,7 @@ KStreamX process( * {@link org.apache.kafka.streams.processor.api.Processor}. Errors in the mapper are captured * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed * {@link org.apache.kafka.streams.processor.api.Processor} + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors * @param the key type of the result stream * @param the value type of the result stream @@ -915,6 +916,7 @@ KErrorStream processCapturingErrors( * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed * {@link org.apache.kafka.streams.processor.api.Processor} * @param errorFilter expression that filters errors which should be thrown and not captured + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors * @param the key type of the result stream * @param the value type of the result stream @@ -937,6 +939,7 @@ KStreamX process( * @param processorSupplier an instance of {@link ProcessorSupplier} that generates a newly constructed * {@link org.apache.kafka.streams.processor.api.Processor} * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors * @param the key type of the result stream * @param the value type of the result stream @@ -954,6 +957,7 @@ KErrorStream processCapturingErrors( * {@link org.apache.kafka.streams.processor.api.Processor} * @param errorFilter expression that filters errors which should be thrown and not captured * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with new key and value as well as captured errors * @param the key type of the result stream * @param the value type of the result stream @@ -976,6 +980,7 @@ KStreamX processValues( * mapper are captured * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured * errors * @param the value type of the result stream @@ -993,6 +998,7 @@ KErrorStream processValuesCapturingErrors( * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} * @param errorFilter expression that filters errors which should be thrown and not captured + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured * errors * @param the value type of the result stream @@ -1016,6 +1022,7 @@ KStreamX processValues( * @param processorSupplier an instance of {@link FixedKeyProcessorSupplier} that generates a newly constructed * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured * errors * @param the value type of the result stream @@ -1034,6 +1041,7 @@ KErrorStream processValuesCapturingErrors( * {@link org.apache.kafka.streams.processor.api.FixedKeyProcessor} * @param errorFilter expression that filters errors which should be thrown and not captured * @param named a {@link Named} config used to name the processor in the topology + * @param stateStoreNames the names of the state store used by the processor * @return a {@link KErrorStream} that contains records with unmodified key and new values as well as captured * errors * @param the value type of the result stream From e687c3eac8eec4e6ad598f9a3cc2f4ec9bb72fb5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 20:51:07 +0100 Subject: [PATCH 33/72] Add tests --- .../bakdata/kafka/KGroupedStreamXTest.java | 78 ++ .../java/com/bakdata/kafka/KStreamXTest.java | 761 ++++++++++++++++-- 2 files changed, 782 insertions(+), 57 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index fb73e72a..f4d2857a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -28,6 +28,7 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Named; import org.junit.jupiter.api.Test; class KGroupedStreamXTest { @@ -97,6 +98,44 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldReduceNamedUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + AutoConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KGroupedStreamX grouped = input.groupByKey( + AutoGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KTableX reduced = grouped.reduce(Long::sum, Named.as("reduce"), + AutoMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + reduced.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldAggregate() { final StreamsApp app = new SimpleApp() { @@ -163,4 +202,43 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.stop(); } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + AutoConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KGroupedStreamX grouped = input.groupByKey( + AutoGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KTableX aggregated = + grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, + Named.as("aggregate"), AutoMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + aggregated.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + topology.stop(); + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 7cc8bca6..6c6a7ced 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -37,6 +37,7 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; @@ -245,6 +246,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldMapNamed() { + final KeyValueMapper> mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn(new KeyValue<>("baz", "qux")); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.map(mapper, Named.as("map")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldMapValues() { final ValueMapper mapper = mock(); @@ -267,6 +290,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldMapValuesNamed() { + final ValueMapper mapper = mock(); + when(mapper.apply("bar")).thenReturn("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.mapValues(mapper, Named.as("map")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldMapValuesWithKey() { final ValueMapperWithKey mapper = mock(); @@ -289,6 +334,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldMapValuesWithKeyNamed() { + final ValueMapperWithKey mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.mapValues(mapper, Named.as("map")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldFlatMap() { final KeyValueMapper>> mapper = mock(); @@ -311,6 +378,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldFlatMapNamed() { + final KeyValueMapper>> mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn(List.of(new KeyValue<>("baz", "qux"))); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.flatMap(mapper, Named.as("flatMap")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldFlatMapValues() { final ValueMapper> mapper = mock(); @@ -333,6 +422,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldFlatMapValuesNamed() { + final ValueMapper> mapper = mock(); + when(mapper.apply("bar")).thenReturn(List.of("baz")); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.flatMapValues(mapper, Named.as("flatMap")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldFlatMapValuesWithKey() { final ValueMapperWithKey> mapper = mock(); @@ -355,6 +466,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldFlatMapValuesWithKeyNamed() { + final ValueMapperWithKey> mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn(List.of("baz")); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.flatMapValues(mapper, Named.as("flatMap")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldProcess() { final ProcessorSupplier processor = () -> new SimpleProcessor<>() { @@ -387,6 +520,38 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldProcessNamed() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + this.forward(inputRecord.withKey("baz").withValue("qux")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX processed = input.process(processor, Named.as("process")); + processed.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldProcessValues() { final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { @@ -420,87 +585,456 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFilter() { - final Predicate predicate = mock(); - when(predicate.test("foo", "bar")).thenReturn(true); - when(predicate.test("foo", "baz")).thenReturn(false); + void shouldProcessValuesNamed() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + this.forward(inputRecord.withValue("baz")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX processed = input.processValues(processor, Named.as("process")); + processed.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFilter() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("foo", "baz")).thenReturn(false); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.filter(predicate).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFilterNamed() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("foo", "baz")).thenReturn(false); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.filter(predicate, Named.as("filter")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFilterNot() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("foo", "baz")).thenReturn(true); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.filterNot(predicate).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldFilterNotNamed() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("foo", "baz")).thenReturn(true); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.filterNot(predicate, Named.as("filterNot")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldSelectKey() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.selectKey((k, v) -> v).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldSelectKeyNamed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.selectKey((k, v) -> v, Named.as("select")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapCapturingErrors() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapCapturingErrorsNamed() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper, Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValuesCapturingErrors() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValuesCapturingErrorsNamed() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMapValuesWithKeyCapturingErrors() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - input.filter(predicate).to("output"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); } }; final TestTopology topology = startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") .expectNextRecord() .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); topology.stop(); } @Test - void shouldFilterNot() { - final Predicate predicate = mock(); - when(predicate.test("foo", "bar")).thenReturn(false); - when(predicate.test("foo", "baz")).thenReturn(true); + void shouldMapValuesWithKeyCapturingErrorsNamed() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - input.filterNot(predicate).to("output"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); } }; final TestTopology topology = startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") .expectNextRecord() .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); topology.stop(); } @Test - void shouldSelectKey() { + void shouldFlatMapCapturingErrors() { + final KeyValueMapper>> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - input.selectKey((k, v) -> v).to("output"); + final KErrorStream processed = + input.flatMapCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); } }; final TestTopology topology = startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") .expectNextRecord() - .hasKey("bar") + .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); topology.stop(); } @Test - void shouldMapCapturingErrors() { - final KeyValueMapper> mapper = mock(); + void shouldFlatMapCapturingErrorsNamed() { + final KeyValueMapper>> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapCapturingErrors(mapper); + input.flatMapCapturingErrors(mapper, Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -529,16 +1063,16 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldMapValuesCapturingErrors() { - final ValueMapper mapper = mock(); + void shouldFlatMapValuesCapturingErrors() { + final ValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); - doReturn("success").when(mapper).apply("baz"); + doReturn(List.of("success")).when(mapper).apply("baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapValuesCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -567,16 +1101,16 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldMapValuesWithKeyCapturingErrors() { - final ValueMapperWithKey mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn("success").when(mapper).apply("foo", "baz"); + void shouldFlatMapValuesCapturingErrorsNamed() { + final ValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn(List.of("success")).when(mapper).apply("baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapValuesCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper, Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -605,16 +1139,16 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFlatMapCapturingErrors() { - final KeyValueMapper>> mapper = mock(); + void shouldFlatMapValuesWithKeyCapturingErrors() { + final ValueMapperWithKey> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + doReturn(List.of("success")).when(mapper).apply("foo", "baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -634,8 +1168,8 @@ public void buildTopology(final TopologyBuilder builder) { topology.input().add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") + .hasKey("foo") + .hasValue("success") .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); @@ -643,16 +1177,16 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFlatMapValuesCapturingErrors() { - final ValueMapper> mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); - doReturn(List.of("success")).when(mapper).apply("baz"); + void shouldFlatMapValuesWithKeyCapturingErrorsNamed() { + final ValueMapperWithKey> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of("success")).when(mapper).apply("foo", "baz"); final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapValuesCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper, Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -681,16 +1215,27 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFlatMapValuesWithKeyCapturingErrors() { - final ValueMapperWithKey> mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn(List.of("success")).when(mapper).apply("foo", "baz"); + void shouldProcessCapturingErrors() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.forward(inputRecord.withKey("success_key").withValue("success_value")); + return; + } + throw new UnsupportedOperationException(); + } + }; final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapValuesCapturingErrors(mapper); + input.processCapturingErrors(processor); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -710,8 +1255,8 @@ public void buildTopology(final TopologyBuilder builder) { topology.input().add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() - .hasKey("foo") - .hasValue("success") + .hasKey("success_key") + .hasValue("success_value") .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); @@ -719,7 +1264,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldProcessCapturingErrors() { + void shouldProcessCapturingErrorsNamed() { final ProcessorSupplier processor = () -> new SimpleProcessor<>() { @Override @@ -739,7 +1284,7 @@ public void process(final Record inputRecord) { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.processCapturingErrors(processor); + input.processCapturingErrors(processor, Named.as("process")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -816,6 +1361,55 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldProcessValuesCapturingErrorsNamed() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.forward(inputRecord.withValue("success")); + return; + } + throw new UnsupportedOperationException(); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.processValuesCapturingErrors(processor, Named.as("process")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldRepartition() { final StreamsApp app = new SimpleApp() { @@ -891,6 +1485,28 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldConvertToTableNamed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KTableX table = input.toTable(Named.as("toTable")); + table.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldConvertToTableUsingMaterialized() { final StreamsApp app = new SimpleApp() { @@ -922,6 +1538,37 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldConvertToTableNamedUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + AutoConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KTableX table = input.toTable(Named.as("toTable"), + AutoMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + table.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldJoin() { final StreamsApp app = new SimpleApp() { From 10adc0cdf067e186d8cf7c84c4b673ec7e2dd962 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 21:06:25 +0100 Subject: [PATCH 34/72] Add tests --- .../com/bakdata/kafka/BranchedKStreamX.java | 2 +- .../com/bakdata/kafka/CogroupedKStreamX.java | 2 +- .../com/bakdata/kafka/KGroupedStreamX.java | 2 +- .../com/bakdata/kafka/KGroupedTableX.java | 2 +- .../main/java/com/bakdata/kafka/KStreamX.java | 2 +- .../main/java/com/bakdata/kafka/KTableX.java | 2 +- .../SessionWindowedCogroupedKStreamX.java | 2 +- .../kafka/SessionWindowedKStreamX.java | 2 +- .../kafka/TimeWindowedCogroupedKStreamX.java | 2 +- .../bakdata/kafka/TimeWindowedKStreamX.java | 2 +- .../com/bakdata/kafka/TopologyBuilder.java | 54 +++---- .../bakdata/kafka/KGroupedStreamXTest.java | 132 ++++++++++++++++++ 12 files changed, 169 insertions(+), 37 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java index 6a0d75b5..d35ea51e 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java @@ -31,7 +31,7 @@ import org.apache.kafka.streams.kstream.Predicate; /** - * Extends the {@code BranchedKStream} interface by adding methods to simplify Serde configuration, error handling, + * Extends the {@link BranchedKStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java index 9ca502d5..a353d9df 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java @@ -38,7 +38,7 @@ import org.apache.kafka.streams.state.KeyValueStore; /** - * Extends the {@code CogroupedKStream} interface by adding methods to simplify Serde configuration, error handling, + * Extends the {@link CogroupedKStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java index f4f8c70a..3d697a3c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java @@ -38,7 +38,7 @@ import org.apache.kafka.streams.state.KeyValueStore; /** - * Extends the {@code KGroupedStream} interface by adding methods to simplify Serde configuration, error handling, + * Extends the {@link KGroupedStream} interface by adding methods to simplify Serde configuration, error handling, * and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java index 9773160c..e59b3764 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java @@ -34,7 +34,7 @@ import org.apache.kafka.streams.state.KeyValueStore; /** - * Extends the {@code KGroupedTable} interface by adding methods to simplify Serde configuration, error handling, and + * Extends the {@link KGroupedTable} interface by adding methods to simplify Serde configuration, error handling, and * topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java index 4deac9f6..689eae23 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java @@ -53,7 +53,7 @@ import org.apache.kafka.streams.state.KeyValueStore; /** - * Extends the {@code KStream} interface by adding methods to simplify Serde configuration, error handling, and topic + * Extends the {@link KStream} interface by adding methods to simplify Serde configuration, error handling, and topic * access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index b5a3fa84..c7d4217d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -42,7 +42,7 @@ import org.apache.kafka.streams.state.KeyValueStore; /** - * Extends the {@code KTable} interface by adding methods to simplify Serde configuration, error handling, and topic + * Extends the {@link KTable} interface by adding methods to simplify Serde configuration, error handling, and topic * access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java index 2b0b95aa..38e6bad0 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java @@ -34,7 +34,7 @@ import org.apache.kafka.streams.state.SessionStore; /** - * Extends the {@code SessionWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, + * Extends the {@link SessionWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java index 616ba980..d004c338 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java @@ -37,7 +37,7 @@ import org.apache.kafka.streams.state.SessionStore; /** - * Extends the {@code SessionWindowedKStream} interface by adding methods to simplify Serde configuration, error + * Extends the {@link SessionWindowedKStream} interface by adding methods to simplify Serde configuration, error * handling, and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java index 92bbc6b5..8a60012c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java @@ -33,7 +33,7 @@ import org.apache.kafka.streams.state.WindowStore; /** - * Extends the {@code TimeWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, + * Extends the {@link TimeWindowedCogroupedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java index 7a33a2e7..f275bf52 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java @@ -36,7 +36,7 @@ import org.apache.kafka.streams.state.WindowStore; /** - * Extends the {@code TimeWindowedKStream} interface by adding methods to simplify Serde configuration, + * Extends the {@link TimeWindowedKStream} interface by adding methods to simplify Serde configuration, * error handling, and topic access * @param type of keys * @param type of values diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index f97982f8..0fefb188 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -114,9 +114,9 @@ public KStreamX stream(final Pattern topicPattern, final AutoConsum } /** - * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics()} + * Create a {@link KStreamX} from all {@link StreamsTopicConfig#getInputTopics()} * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @return a {@link KStreamX} for all {@link StreamsTopicConfig#getInputTopics()} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) @@ -126,9 +126,9 @@ public KStreamX streamInput(final Consumed consumed) { } /** - * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics()} + * Create a {@link KStreamX} from all {@link StreamsTopicConfig#getInputTopics()} * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * @return a {@link KStreamX} for all {@link StreamsTopicConfig#getInputTopics()} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) @@ -138,8 +138,8 @@ public KStreamX streamInput(final AutoConsumed consumed) { } /** - * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics()} - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics()} + * Create a {@link KStreamX} from all {@link StreamsTopicConfig#getInputTopics()} + * @return a {@link KStreamX} for all {@link StreamsTopicConfig#getInputTopics()} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Collection) @@ -149,10 +149,10 @@ public KStreamX streamInput() { } /** - * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics(String)} + * Create a {@link KStreamX} from all {@link StreamsTopicConfig#getInputTopics(String)} * @param label label of input topics * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics(String)} + * @return a {@link KStreamX} for all {@link StreamsTopicConfig#getInputTopics(String)} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) @@ -162,10 +162,10 @@ public KStreamX streamInput(final String label, final Consumed type of keys * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) @@ -175,9 +175,9 @@ public KStreamX streamInput(final String label, final AutoConsumed< } /** - * Create a {@code KStream} from all {@link StreamsTopicConfig#getInputTopics(String)} + * Create a {@link KStreamX} from all {@link StreamsTopicConfig#getInputTopics(String)} * @param label label of input topics - * @return a {@code KStream} for all {@link StreamsTopicConfig#getInputTopics(String)} + * @return a {@link KStreamX} for all {@link StreamsTopicConfig#getInputTopics(String)} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Collection) @@ -187,9 +187,9 @@ public KStreamX streamInput(final String label) { } /** - * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern()} + * Create a {@link KStreamX} from all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} + * @return a {@link KStreamX} for all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) @@ -199,9 +199,9 @@ public KStreamX streamInputPattern(final Consumed consumed) { } /** - * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern()} + * Create a {@link KStreamX} from all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} + * @return a {@link KStreamX} for all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) @@ -211,8 +211,8 @@ public KStreamX streamInputPattern(final AutoConsumed consume } /** - * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern()} - * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern()} + * Create a {@link KStreamX} from all topics matching {@link StreamsTopicConfig#getInputPattern()} + * @return a {@link KStreamX} for all topics matching {@link StreamsTopicConfig#getInputPattern()} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Pattern) @@ -222,10 +222,10 @@ public KStreamX streamInputPattern() { } /** - * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * Create a {@link KStreamX} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param label label of input pattern * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * @return a {@link KStreamX} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) @@ -235,10 +235,10 @@ public KStreamX streamInputPattern(final String label, final Consum } /** - * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * Create a {@link KStreamX} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param label label of input pattern * @param consumed define optional parameters for streaming topics - * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * @return a {@link KStreamX} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) @@ -249,9 +249,9 @@ public KStreamX streamInputPattern(final String label, } /** - * Create a {@code KStream} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * Create a {@link KStreamX} from all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param label label of input pattern - * @return a {@code KStream} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} + * @return a {@link KStreamX} for all topics matching {@link StreamsTopicConfig#getInputPattern(String)} * @param type of keys * @param type of values * @see StreamsBuilder#stream(Pattern) @@ -278,8 +278,8 @@ public EffectiveAppConfiguration createEffectiveConfiguratio } /** - * Create a {@code StreamsContext} to wrap Kafka Streams interfaces - * @return {@code StreamsContext} + * Create a {@link StreamsContext} to wrap Kafka Streams interfaces + * @return {@link StreamsContext} */ public StreamsContext getContext() { return new StreamsContext(this.topics, this.createConfigurator()); @@ -287,7 +287,7 @@ public StreamsContext getContext() { /** * Create stores using application context to lazily configures Serdes - * @return {@code AutoStores} + * @return {@link AutoStores} */ public AutoStores stores() { return new AutoStores(this.createConfigurator()); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index f4d2857a..15aaa022 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -33,6 +33,138 @@ class KGroupedStreamXTest { + @Test + void shouldCount() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final KTableX counted = grouped.count(); + counted.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldCountNamed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final KTableX counted = grouped.count(Named.as("count")); + counted.toStream().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldCountUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + AutoConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KGroupedStreamX grouped = input.groupByKey( + AutoGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KTableX counted = + grouped.count(AutoMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + counted.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldCountNamedUsingMaterialized() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + AutoConsumed.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KGroupedStreamX grouped = input.groupByKey( + AutoGrouped.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + final KTableX counted = + grouped.count(Named.as("count"), AutoMaterialized.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + counted.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), + Preconfigured.create(Serdes.Long()))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + topology.stop(); + } + @Test void shouldReduce() { final StreamsApp app = new SimpleApp() { From baefa303afdcc087bdbf023194e10d14215b0207 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 21:26:47 +0100 Subject: [PATCH 35/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 238 +++++++++++++++++- 1 file changed, 237 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 6c6a7ced..39402fe7 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -27,20 +27,24 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; import com.bakdata.fluent_kafka_streams_tests.TestTopology; -import io.vavr.collection.List; import java.time.Duration; +import java.util.List; import java.util.Map; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.ForeachAction; import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.processor.api.FixedKeyProcessor; import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; @@ -49,6 +53,12 @@ import org.apache.kafka.streams.processor.api.ProcessorContext; import org.apache.kafka.streams.processor.api.ProcessorSupplier; import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.junit.jupiter.MockitoExtension; @@ -57,8 +67,12 @@ @ExtendWith(MockitoExtension.class) @MockitoSettings(strictness = Strictness.STRICT_STUBS) +@ExtendWith(SoftAssertionsExtension.class) class KStreamXTest { + @InjectSoftAssertions + private SoftAssertions softly; + static TestTopology startApp(final StreamsApp app, final StreamsTopicConfig topicConfig) { final ConfiguredStreamsApp configuredStreamsApp = new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig)); @@ -552,6 +566,66 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldProcessUsingStore() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.getStreamsBuilder().addStateStore(store); + final KStreamX input = builder.stream("input"); + input.process(processor, "my-store"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + topology.stop(); + } + + @Test + void shouldProcessNamedUsingStore() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.getStreamsBuilder().addStateStore(store); + final KStreamX input = builder.stream("input"); + input.process(processor, Named.as("process"), "my-store"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + topology.stop(); + } + @Test void shouldProcessValues() { final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { @@ -616,6 +690,66 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldProcessValuesUsingStore() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.getStreamsBuilder().addStateStore(store); + final KStreamX input = builder.stream("input"); + input.processValues(processor, "my-store"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + topology.stop(); + } + + @Test + void shouldProcessValuesNamedUsingStore() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.getStreamsBuilder().addStateStore(store); + final KStreamX input = builder.stream("input"); + input.processValues(processor, Named.as("process"), "my-store"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + topology.stop(); + } + @Test void shouldFilter() { final Predicate predicate = mock(); @@ -2163,6 +2297,100 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldMerge() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input1 = builder.stream("input1"); + final KStreamX input2 = builder.stream("input2"); + final KStreamX grouped = input1.merge(input2); + grouped.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input1") + .add("foo", "bar"); + topology.input("input2") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldMergeNamed() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input1 = builder.stream("input1"); + final KStreamX input2 = builder.stream("input2"); + final KStreamX grouped = input1.merge(input2, Named.as("merge")); + grouped.to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input1") + .add("foo", "bar"); + topology.input("input2") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldDoForEach() { + final ForeachAction action = mock(); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.foreach(action); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + topology.stop(); + } + + @Test + void shouldDoForEachNamed() { + final ForeachAction action = mock(); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.foreach(action, Named.as("forEach")); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + topology.stop(); + } + private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; @@ -2175,6 +2403,10 @@ protected void forward(final Record outputRecord this.context.forward(outputRecord); } + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + } private abstract static class SimpleFixedKeyProcessor implements FixedKeyProcessor { @@ -2189,5 +2421,9 @@ protected void forward(final FixedKeyRecord outpu this.context.forward(outputRecord); } + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + } } From 86935e4dfad5da985b90c303a9327cb33abca7d2 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 21:43:23 +0100 Subject: [PATCH 36/72] Add methods to TopologyBuilder --- .../com/bakdata/kafka/TopologyBuilder.java | 140 ++++++++++++++++++ .../java/com/bakdata/kafka/KStreamXTest.java | 8 +- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index 0fefb188..05667802 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -30,9 +30,15 @@ import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Value; +import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.kstream.GlobalKTable; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; /** * Provides all runtime configurations and supports building a {@link Topology} of a {@link StreamsApp} @@ -260,6 +266,140 @@ public KStreamX streamInputPattern(final String label) { return this.stream(this.topics.getInputPattern(label)); } + /** + * @see StreamsBuilder#table(String) + */ + public KTableX table(final String topic) { + return this.getContext().wrap(this.streamsBuilder.table(topic)); + } + + /** + * @see StreamsBuilder#table(String, Consumed) + */ + public KTableX table(final String topic, final Consumed consumed) { + return this.getContext().wrap(this.streamsBuilder.table(topic, consumed)); + } + + /** + * @see StreamsBuilder#table(String, Consumed) + */ + public KTableX table(final String topic, final AutoConsumed consumed) { + return this.table(topic, consumed.configure(this.createConfigurator())); + } + + /** + * @see StreamsBuilder#table(String, Materialized) + */ + public KTableX table(final String topic, + final Materialized> materialized) { + return this.getContext().wrap(this.streamsBuilder.table(topic, materialized)); + } + + /** + * @see StreamsBuilder#table(String, Materialized) + */ + public KTableX table(final String topic, + final AutoMaterialized> materialized) { + return this.table(topic, materialized.configure(this.createConfigurator())); + } + + /** + * @see StreamsBuilder#table(String, Consumed, Materialized) + */ + public KTableX table(final String topic, final Consumed consumed, + final Materialized> materialized) { + return this.getContext().wrap(this.streamsBuilder.table(topic, consumed, materialized)); + } + + /** + * @see StreamsBuilder#table(String, Consumed, Materialized) + */ + public KTableX table(final String topic, final AutoConsumed consumed, + final AutoMaterialized> materialized) { + final Configurator configurator = this.createConfigurator(); + return this.table(topic, consumed.configure(configurator), materialized.configure(configurator)); + } + + /** + * @see StreamsBuilder#globalTable(String) + */ + public GlobalKTable globalTable(final String topic) { + return this.streamsBuilder.globalTable(topic); + } + + /** + * @see StreamsBuilder#globalTable(String, Consumed) + */ + public GlobalKTable globalTable(final String topic, final Consumed consumed) { + return this.streamsBuilder.globalTable(topic, consumed); + } + + /** + * @see StreamsBuilder#globalTable(String, Consumed) + */ + public GlobalKTable globalTable(final String topic, final AutoConsumed consumed) { + return this.globalTable(topic, consumed.configure(this.createConfigurator())); + } + + /** + * @see StreamsBuilder#globalTable(String, Materialized) + */ + public GlobalKTable globalTable(final String topic, + final Materialized> materialized) { + return this.streamsBuilder.globalTable(topic, materialized); + } + + /** + * @see StreamsBuilder#globalTable(String, Materialized) + */ + public GlobalKTable globalTable(final String topic, + final AutoMaterialized> materialized) { + return this.globalTable(topic, materialized.configure(this.createConfigurator())); + } + + /** + * @see StreamsBuilder#globalTable(String, Consumed, Materialized) + */ + public GlobalKTable globalTable(final String topic, final Consumed consumed, + final Materialized> materialized) { + return this.streamsBuilder.globalTable(topic, consumed, materialized); + } + + /** + * @see StreamsBuilder#globalTable(String, Consumed, Materialized) + */ + public GlobalKTable globalTable(final String topic, final AutoConsumed consumed, + final AutoMaterialized> materialized) { + final Configurator configurator = this.createConfigurator(); + return this.globalTable(topic, consumed.configure(configurator), materialized.configure(configurator)); + } + + /** + * @see StreamsBuilder#addStateStore(StoreBuilder) + */ + public TopologyBuilder addStateStore(final StoreBuilder builder) { + this.streamsBuilder.addStateStore(builder); + return this; + } + + /** + * @see StreamsBuilder#addGlobalStore(StoreBuilder, String, Consumed, ProcessorSupplier) + */ + public TopologyBuilder addGlobalStore(final StoreBuilder storeBuilder, final String topic, + final Consumed consumed, final ProcessorSupplier stateUpdateSupplier) { + this.streamsBuilder.addGlobalStore(storeBuilder, topic, consumed, stateUpdateSupplier); + return this; + } + + /** + * @see StreamsBuilder#addGlobalStore(StoreBuilder, String, Consumed, ProcessorSupplier) + */ + public TopologyBuilder addGlobalStore(final StoreBuilder storeBuilder, final String topic, + final AutoConsumed consumed, final ProcessorSupplier stateUpdateSupplier) { + return this.addGlobalStore(storeBuilder, topic, consumed.configure(this.createConfigurator()), + stateUpdateSupplier); + } + /** * Create {@code Configurator} to configure {@link org.apache.kafka.common.serialization.Serde} and * {@link org.apache.kafka.common.serialization.Serializer} using {@link #kafkaProperties}. diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 39402fe7..8e752bdc 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -582,7 +582,7 @@ public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); - builder.getStreamsBuilder().addStateStore(store); + builder.addStateStore(store); final KStreamX input = builder.stream("input"); input.process(processor, "my-store"); } @@ -612,7 +612,7 @@ public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); - builder.getStreamsBuilder().addStateStore(store); + builder.addStateStore(store); final KStreamX input = builder.stream("input"); input.process(processor, Named.as("process"), "my-store"); } @@ -706,7 +706,7 @@ public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); - builder.getStreamsBuilder().addStateStore(store); + builder.addStateStore(store); final KStreamX input = builder.stream("input"); input.processValues(processor, "my-store"); } @@ -736,7 +736,7 @@ public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); - builder.getStreamsBuilder().addStateStore(store); + builder.addStateStore(store); final KStreamX input = builder.stream("input"); input.processValues(processor, Named.as("process"), "my-store"); } From 2e0f96d309a7eb982438bb8f849e74bdec569fa4 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 22:20:07 +0100 Subject: [PATCH 37/72] Extend Branched --- .../com/bakdata/kafka/BranchedKStreamX.java | 10 + .../bakdata/kafka/BranchedKStreamXImpl.java | 10 + .../java/com/bakdata/kafka/BranchedX.java | 118 ++++++++++++ .../bakdata/kafka/BranchedKStreamXTest.java | 175 ++++++++++++++++++ .../java/com/bakdata/kafka/KStreamXTest.java | 48 +++++ 5 files changed, 361 insertions(+) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java index d35ea51e..8a8cb317 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java @@ -44,12 +44,22 @@ public interface BranchedKStreamX extends BranchedKStream { @Override BranchedKStreamX branch(Predicate predicate, Branched branched); + /** + * @see #branch(Predicate, Branched) + */ + BranchedKStreamX branch(Predicate predicate, BranchedX branched); + @Override Map> defaultBranch(); @Override Map> defaultBranch(Branched branched); + /** + * @see #defaultBranch(Branched) + */ + Map> defaultBranch(BranchedX branched); + @Override Map> noDefaultBranch(); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java index e210fb9e..74e45627 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java @@ -51,6 +51,11 @@ public BranchedKStreamX branch(final Predicate predi return this.context.wrap(this.wrapped.branch(predicate, branched)); } + @Override + public BranchedKStreamX branch(final Predicate predicate, final BranchedX branched) { + return this.branch(predicate, branched.convert(this.context)); + } + @Override public Map> defaultBranch() { return this.wrap(this.wrapped.defaultBranch()); @@ -61,6 +66,11 @@ public Map> defaultBranch(final Branched branched) { return this.wrap(this.wrapped.defaultBranch(branched)); } + @Override + public Map> defaultBranch(final BranchedX branched) { + return this.defaultBranch(branched.convert(this.context)); + } + @Override public Map> noDefaultBranch() { return this.wrap(this.wrapped.noDefaultBranch()); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java new file mode 100644 index 00000000..f2075547 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java @@ -0,0 +1,118 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Consumer; +import java.util.function.Function; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.streams.kstream.Branched; +import org.apache.kafka.streams.kstream.KStream; + +/** + * Extends the {@link org.apache.kafka.streams.kstream.Branched} interface by adding methods to simplify Serde + * configuration, error handling, + * and topic access + * @param type of keys + * @param type of values + */ +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) +public abstract class BranchedX { + + @Getter(AccessLevel.PROTECTED) + private final String name; + + /** + * @see Branched#withConsumer(Consumer) + */ + public static BranchedX withConsumer(final Consumer> chain) { + return new ConsumerBranchedX<>(null, chain); + } + + /** + * @see Branched#withConsumer(Consumer, String) + */ + public static BranchedX withConsumer(final Consumer> chain, final String name) { + return new ConsumerBranchedX<>(name, chain); + } + + /** + * @see Branched#withFunction(Function) + */ + public static BranchedX withFunction( + final Function, ? extends KStream> chain) { + return new FunctionBranchedX<>(null, chain); + } + + /** + * @see Branched#withFunction(Function, String) + */ + public static BranchedX withFunction( + final Function, ? extends KStream> chain, final String name) { + return new FunctionBranchedX<>(name, chain); + } + + abstract Branched convert(StreamsContext context); + + private static final class ConsumerBranchedX extends BranchedX { + private final @NonNull Consumer> chain; + + private ConsumerBranchedX(final String name, final Consumer> chain) { + super(name); + this.chain = chain; + } + + @Override + Branched convert(final StreamsContext context) { + return Branched.withConsumer(this.asConsumer(context), this.getName()); + } + + private Consumer> asConsumer(final StreamsContext context) { + return stream -> this.chain.accept(context.wrap(stream)); + } + } + + private static final class FunctionBranchedX extends BranchedX { + private final @NonNull Function, ? extends KStream> chain; + + private FunctionBranchedX(final String name, + final Function, ? extends KStream> chain) { + super(name); + this.chain = chain; + } + + @Override + Branched convert(final StreamsContext context) { + return Branched.withFunction(this.asFunction(context), this.getName()); + } + + private Function, KStream> asFunction(final StreamsContext context) { + return stream -> this.chain.apply(context.wrap(stream)); + } + } + +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java new file mode 100644 index 00000000..d5bf35a0 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java @@ -0,0 +1,175 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static com.bakdata.kafka.KStreamXTest.startApp; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.util.Map; +import org.apache.kafka.streams.kstream.Branched; +import org.apache.kafka.streams.kstream.KStream; +import org.junit.jupiter.api.Test; + +class BranchedKStreamXTest { + + @Test + void shouldBranch() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .branch((k, v) -> "foo".equals(k)) + .noDefaultBranch(); + branches.values().iterator().next().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldBranchBranched() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.split() + .branch((k, v) -> "foo".equals(k), Branched.withConsumer(branch -> branch.to("output"))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldBranchBranchedX() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.split() + .branch((k, v) -> "foo".equals(k), BranchedX.withConsumer(KStreamX::toOutputTopic)); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldUseDefaultBranch() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .defaultBranch(); + branches.values().iterator().next().to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldUseDefaultBranched() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.split() + .defaultBranch(Branched.withConsumer(branch -> branch.to("output"))); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldUseDefaultBranchedX() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.split() + .defaultBranch(BranchedX.withConsumer(KStreamX::toOutputTopic)); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 8e752bdc..f7fe0f66 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -2391,6 +2391,54 @@ public void buildTopology(final TopologyBuilder builder) { topology.stop(); } + @Test + void shouldPeek() { + final ForeachAction action = mock(); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.peek(action).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + topology.stop(); + } + + @Test + void shouldPeekNamed() { + final ForeachAction action = mock(); + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.peek(action, Named.as("peek")).to("output"); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder().build()); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + topology.stop(); + } + private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; From b5b9f6d591a093678b4f0185f57f20ce155563c9 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 22:51:57 +0100 Subject: [PATCH 38/72] Extend Branched --- .../com/bakdata/kafka/BranchedKStreamX.java | 29 +++++- .../bakdata/kafka/BranchedKStreamXImpl.java | 57 +++++++++-- .../bakdata/kafka/BranchedKStreamXTest.java | 99 +++++++++++++++++-- 3 files changed, 168 insertions(+), 17 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java index 8a8cb317..cb4adade 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java @@ -49,17 +49,44 @@ public interface BranchedKStreamX extends BranchedKStream { */ BranchedKStreamX branch(Predicate predicate, BranchedX branched); + /** + * @deprecated Use {@link #defaultBranchX()} + */ + @Deprecated @Override Map> defaultBranch(); + /** + * @see BranchedKStream#defaultBranch() + */ + Map> defaultBranchX(); + + /** + * @deprecated Use {@link #defaultBranchX(Branched)} + */ + @Deprecated @Override Map> defaultBranch(Branched branched); + /** + * @see BranchedKStream#defaultBranch(Branched) + */ + Map> defaultBranchX(Branched branched); + /** * @see #defaultBranch(Branched) */ - Map> defaultBranch(BranchedX branched); + Map> defaultBranch(BranchedX branched); + /** + * @deprecated Use {@link #noDefaultBranchX()} + */ + @Deprecated @Override Map> noDefaultBranch(); + + /** + * @see BranchedKStream#noDefaultBranch() + */ + Map> noDefaultBranchX(); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java index 74e45627..0a89b6a1 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java @@ -26,6 +26,7 @@ import java.util.Map; import java.util.Map.Entry; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -40,6 +41,14 @@ class BranchedKStreamXImpl implements BranchedKStreamX { private final @NonNull BranchedKStream wrapped; private final @NonNull StreamsContext context; + private static Map wrap( + final Map> streamMap, + final Function>, ? extends T> wrapValue) { + return streamMap.entrySet() + .stream() + .collect(Collectors.toMap(Entry::getKey, wrapValue)); + } + @Override public BranchedKStreamX branch(final Predicate predicate) { return this.context.wrap(this.wrapped.branch(predicate)); @@ -52,34 +61,64 @@ public BranchedKStreamX branch(final Predicate predi } @Override - public BranchedKStreamX branch(final Predicate predicate, final BranchedX branched) { + public BranchedKStreamX branch(final Predicate predicate, + final BranchedX branched) { return this.branch(predicate, branched.convert(this.context)); } @Override public Map> defaultBranch() { - return this.wrap(this.wrapped.defaultBranch()); + return this.wrap(this.defaultBranchInternal()); + } + + @Override + public Map> defaultBranchX() { + return this.wrapX(this.defaultBranchInternal()); } @Override public Map> defaultBranch(final Branched branched) { - return this.wrap(this.wrapped.defaultBranch(branched)); + return this.wrap(this.defaultBranchInternal(branched)); + } + + @Override + public Map> defaultBranchX(final Branched branched) { + return this.wrapX(this.defaultBranchInternal(branched)); } @Override - public Map> defaultBranch(final BranchedX branched) { - return this.defaultBranch(branched.convert(this.context)); + public Map> defaultBranch(final BranchedX branched) { + return this.wrapX(this.defaultBranchInternal(branched.convert(this.context))); } @Override public Map> noDefaultBranch() { - return this.wrap(this.wrapped.noDefaultBranch()); + return this.wrap(this.noDefaultBranchInternal()); + } + + @Override + public Map> noDefaultBranchX() { + return this.wrapX(this.noDefaultBranchInternal()); + } + + private Map> noDefaultBranchInternal() { + return this.wrapped.noDefaultBranch(); + } + + private Map> defaultBranchInternal() { + return this.wrapped.defaultBranch(); + } + + private Map> defaultBranchInternal(final Branched branched) { + return this.wrapped.defaultBranch(branched); } private Map> wrap(final Map> streamMap) { - return streamMap.entrySet() - .stream() - .collect(Collectors.toMap(Entry::getKey, this::wrapValue)); + return BranchedKStreamXImpl.>wrap(streamMap, this::wrapValue); + } + + private Map> wrapX(final Map> streamMap) { + return BranchedKStreamXImpl.>wrap(streamMap, this::wrapValue); } private KStreamX wrapValue(final Entry> entry) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java index d5bf35a0..8a56fd28 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java @@ -28,6 +28,7 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.util.Map; +import java.util.function.Function; import org.apache.kafka.streams.kstream.Branched; import org.apache.kafka.streams.kstream.KStream; import org.junit.jupiter.api.Test; @@ -87,7 +88,8 @@ void shouldBranchBranchedX() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.split() - .branch((k, v) -> "foo".equals(k), BranchedX.withConsumer(KStreamX::toOutputTopic)); + .branch((k, v) -> "foo".equals(k), BranchedX.withConsumer(KStreamX::toOutputTopic)) + .defaultBranch(Branched.withConsumer(s -> s.to("default_output"))); } }; final TestTopology topology = @@ -95,12 +97,18 @@ public void buildTopology(final TopologyBuilder builder) { .outputTopic("output") .build()); topology.input() - .add("foo", "bar"); - topology.streamOutput() + .add("foo", "bar") + .add("baz", "qux"); + topology.streamOutput("output") .expectNextRecord() .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); + topology.streamOutput("default_output") + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); topology.stop(); } @@ -128,7 +136,32 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldUseDefaultBranched() { + void shouldUseDefaultBranchX() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .defaultBranchX(); + branches.values().iterator().next().toOutputTopic(); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldUseDefaultBranchBranched() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -150,13 +183,65 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldUseDefaultBranchedX() { + void shouldUseDefaultBranchXBranched() { final StreamsApp app = new SimpleApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - input.split() - .defaultBranch(BranchedX.withConsumer(KStreamX::toOutputTopic)); + final Map> branches = input.split() + .defaultBranchX(Branched.withFunction(Function.identity())); + branches.values().iterator().next().toOutputTopic(); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldUseDefaultBranchBranchedX() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .defaultBranch(BranchedX.withFunction(Function.identity())); + branches.values().iterator().next().toOutputTopic(); + } + }; + final TestTopology topology = + startApp(app, StreamsTopicConfig.builder() + .outputTopic("output") + .build()); + topology.input() + .add("foo", "bar"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.stop(); + } + + @Test + void shouldUseNoDefaultBranchX() { + final StreamsApp app = new SimpleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .branch((k, v) -> "foo".equals(k)) + .noDefaultBranchX(); + branches.values().iterator().next().toOutputTopic(); } }; final TestTopology topology = From 207c6b50de7aea31cdeb50b0540978d70176e985 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 23:07:02 +0100 Subject: [PATCH 39/72] Extend Branched --- .../src/main/java/com/bakdata/kafka/BranchedKStreamX.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java index cb4adade..8878fc20 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java @@ -52,7 +52,7 @@ public interface BranchedKStreamX extends BranchedKStream { /** * @deprecated Use {@link #defaultBranchX()} */ - @Deprecated + @Deprecated(since = "3.6.0") @Override Map> defaultBranch(); @@ -64,7 +64,7 @@ public interface BranchedKStreamX extends BranchedKStream { /** * @deprecated Use {@link #defaultBranchX(Branched)} */ - @Deprecated + @Deprecated(since = "3.6.0") @Override Map> defaultBranch(Branched branched); @@ -81,7 +81,7 @@ public interface BranchedKStreamX extends BranchedKStream { /** * @deprecated Use {@link #noDefaultBranchX()} */ - @Deprecated + @Deprecated(since = "3.6.0") @Override Map> noDefaultBranch(); From 104354d6d524b6573393dff26ae0299b7d4ae007 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Fri, 14 Feb 2025 23:15:26 +0100 Subject: [PATCH 40/72] Extend Branched --- .../src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java index 8a56fd28..4dd39e75 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java @@ -89,7 +89,7 @@ public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.split() .branch((k, v) -> "foo".equals(k), BranchedX.withConsumer(KStreamX::toOutputTopic)) - .defaultBranch(Branched.withConsumer(s -> s.to("default_output"))); + .defaultBranchX(Branched.withConsumer(s -> s.to("default_output"))); } }; final TestTopology topology = From e69649010332c789479e21de7ca4d558decf761a Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 06:31:26 +0100 Subject: [PATCH 41/72] Rename and test --- gradle.properties | 2 +- .../java/com/bakdata/kafka/AutoConsumed.java | 98 - .../java/com/bakdata/kafka/AutoGrouped.java | 85 - .../java/com/bakdata/kafka/AutoJoined.java | 103 - .../com/bakdata/kafka/AutoMaterialized.java | 139 - .../java/com/bakdata/kafka/AutoProduced.java | 87 - .../com/bakdata/kafka/AutoRepartitioned.java | 125 - .../com/bakdata/kafka/AutoStreamJoined.java | 144 - .../com/bakdata/kafka/BranchedKStreamX.java | 6 +- .../bakdata/kafka/BranchedKStreamXImpl.java | 4 +- .../java/com/bakdata/kafka/BranchedX.java | 80 +- .../com/bakdata/kafka/CogroupedKStreamX.java | 4 +- .../bakdata/kafka/CogroupedStreamXImpl.java | 4 +- .../java/com/bakdata/kafka/ConsumedX.java | 181 ++ .../main/java/com/bakdata/kafka/GroupedX.java | 135 + .../main/java/com/bakdata/kafka/JoinedX.java | 179 ++ .../com/bakdata/kafka/KGroupedStreamX.java | 12 +- .../bakdata/kafka/KGroupedStreamXImpl.java | 12 +- .../com/bakdata/kafka/KGroupedTableX.java | 12 +- .../com/bakdata/kafka/KGroupedTableXImpl.java | 12 +- .../main/java/com/bakdata/kafka/KStreamX.java | 56 +- .../java/com/bakdata/kafka/KStreamXImpl.java | 40 +- .../main/java/com/bakdata/kafka/KTableX.java | 50 +- .../java/com/bakdata/kafka/KTableXImpl.java | 46 +- .../java/com/bakdata/kafka/MaterializedX.java | 238 ++ .../java/com/bakdata/kafka/ModifierChain.java | 62 + .../java/com/bakdata/kafka/ProducedX.java | 149 + .../com/bakdata/kafka/RepartitionedX.java | 180 ++ .../SessionWindowedCogroupedKStreamX.java | 4 +- .../SessionWindowedCogroupedStreamXImpl.java | 4 +- .../kafka/SessionWindowedKStreamX.java | 12 +- .../kafka/SessionWindowedStreamXImpl.java | 12 +- .../kafka/{AutoStores.java => StoresX.java} | 54 +- .../java/com/bakdata/kafka/StreamJoinedX.java | 266 ++ .../kafka/TimeWindowedCogroupedKStreamX.java | 4 +- .../TimeWindowedCogroupedStreamXImpl.java | 4 +- .../bakdata/kafka/TimeWindowedKStreamX.java | 12 +- .../kafka/TimeWindowedStreamXImpl.java | 12 +- .../com/bakdata/kafka/TopologyBuilder.java | 42 +- .../bakdata/kafka/BranchedKStreamXTest.java | 223 +- .../bakdata/kafka/KGroupedStreamXTest.java | 446 ++- .../java/com/bakdata/kafka/KStreamXTest.java | 2474 ++++++++--------- .../java/com/bakdata/kafka/KTableXTest.java | 804 ++++++ .../kafka/{SimpleApp.java => StringApp.java} | 11 +- .../java/com/bakdata/kafka/TestHelper.java | 40 + .../bakdata/kafka/TopologyBuilderTest.java | 492 +++- .../MirrorWithNonDefaultSerde.java | 8 +- 47 files changed, 4354 insertions(+), 2815 deletions(-) delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java delete mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/GroupedX.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/JoinedX.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/MaterializedX.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ModifierChain.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducedX.java create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/RepartitionedX.java rename streams-bootstrap-core/src/main/java/com/bakdata/kafka/{AutoStores.java => StoresX.java} (67%) create mode 100644 streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamJoinedX.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java rename streams-bootstrap-core/src/test/java/com/bakdata/kafka/{SimpleApp.java => StringApp.java} (79%) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java diff --git a/gradle.properties b/gradle.properties index fd753534..e6d3c698 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ org.gradle.parallel=true kafkaVersion=3.8.1 testContainersVersion=1.20.4 confluentVersion=7.8.0 -fluentKafkaVersion=3.0.0 +fluentKafkaVersion=3.0.1-SNAPSHOT junitVersion=5.11.4 mockitoVersion=5.15.2 assertJVersion=3.27.2 diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java deleted file mode 100644 index e7dc4b62..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoConsumed.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.Topology.AutoOffsetReset; -import org.apache.kafka.streams.kstream.Consumed; -import org.apache.kafka.streams.processor.TimestampExtractor; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Consumed} using {@link Configurator} - * @param type of keys - * @param type of values - * @see Consumed - */ -@With -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoConsumed { - - private final @NonNull Preconfigured> keySerde; - private final @NonNull Preconfigured> valueSerde; - private final TimestampExtractor timestampExtractor; - private final AutoOffsetReset offsetResetPolicy; - private final String name; - - /** - * Create an instance of {@code AutoConsumed} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code AutoConsumed} - * @param type of keys - * @param type of values - */ - public static AutoConsumed keySerde(final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde()); - } - - /** - * Create an instance of {@code AutoConsumed} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code AutoConsumed} - * @param type of keys - * @param type of values - */ - public static AutoConsumed valueSerde(final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see Consumed#with(Serde, Serde) - */ - public static AutoConsumed with(final Preconfigured> keySerde, - final Preconfigured> valueSerde) { - return new AutoConsumed<>(keySerde, valueSerde, null, null, null); - } - - /** - * @see Consumed#as(String) - */ - public static AutoConsumed as(final String processorName) { - return new AutoConsumed<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, - processorName); - } - - Consumed configure(final Configurator configurator) { - return Consumed.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) - .withOffsetResetPolicy(this.offsetResetPolicy) - .withTimestampExtractor(this.timestampExtractor); - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java deleted file mode 100644 index 6c383278..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoGrouped.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.kstream.Grouped; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Grouped} using {@link Configurator} - * @param type of keys - * @param type of values - * @see Grouped - */ -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoGrouped { - - @With - private final @NonNull Preconfigured> keySerde; - @With - private final @NonNull Preconfigured> valueSerde; - @With - private final String name; - - /** - * @see Grouped#keySerde(Serde) - */ - public static AutoGrouped keySerde(final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde()); - } - - /** - * @see Grouped#valueSerde(Serde) - */ - public static AutoGrouped valueSerde(final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see Grouped#with(Serde, Serde) - */ - public static AutoGrouped with(final Preconfigured> keySerde, - final Preconfigured> valueSerde) { - return new AutoGrouped<>(keySerde, valueSerde, null); - } - - /** - * @see Grouped#as(String) - */ - public static AutoGrouped as(final String name) { - return new AutoGrouped<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), name); - } - - Grouped configure(final Configurator configurator) { - return Grouped.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)); - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java deleted file mode 100644 index a7087799..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoJoined.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import java.time.Duration; -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.kstream.Joined; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Joined} using {@link Configurator} - * @param type of keys - * @param this value type - * @param other value type - * @see Joined - */ -@With -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoJoined { - - private final @NonNull Preconfigured> keySerde; - private final @NonNull Preconfigured> valueSerde; - private final @NonNull Preconfigured> otherValueSerde; - private final String name; - private final Duration gracePeriod; - - /** - * @see Joined#keySerde(Serde) - */ - public static AutoJoined keySerde( - final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); - } - - /** - * @see Joined#valueSerde(Serde) - */ - public static AutoJoined valueSerde( - final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); - } - - /** - * @see Joined#otherValueSerde(Serde) - */ - public static AutoJoined otherValueSerde( - final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see Joined#with(Serde, Serde, Serde) - */ - public static AutoJoined with( - final Preconfigured> keySerde, - final Preconfigured> valueSerde, - final Preconfigured> otherValueSerde) { - return new AutoJoined<>(keySerde, valueSerde, otherValueSerde, null, null); - } - - /** - * @see Joined#as(String) - */ - public static AutoJoined as(final String name) { - return new AutoJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde(), name, null); - } - - Joined configure(final Configurator configurator) { - return Joined.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) - .withOtherValueSerde(configurator.configureForValues(this.otherValueSerde)) - .withName(this.name) - .withGracePeriod(this.gracePeriod); - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java deleted file mode 100644 index d434554c..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoMaterialized.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.kstream.Materialized; -import org.apache.kafka.streams.processor.StateStore; -import org.apache.kafka.streams.state.DslStoreSuppliers; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Materialized} using {@link Configurator} - * @param type of keys - * @param type of values - * @param type of state store - * @see Materialized - */ -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoMaterialized { - - @With - private final @NonNull Preconfigured> keySerde; - @With - private final @NonNull Preconfigured> valueSerde; - private final String storeName; - @With - private final Duration retention; - @With - private final DslStoreSuppliers storeType; - private final Map topicConfig; - private final boolean loggingEnabled; - private final boolean cachingEnabled; - - /** - * Create an instance of {@code AutoMaterialized} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code AutoMaterialized} - * @param type of keys - * @param type of values - * @param type of state store - */ - public static AutoMaterialized keySerde( - final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde()); - } - - /** - * Create an instance of {@code AutoMaterialized} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code AutoMaterialized} - * @param type of keys - * @param type of values - * @param type of state store - */ - public static AutoMaterialized valueSerde( - final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see Materialized#with(Serde, Serde) - */ - public static AutoMaterialized with( - final Preconfigured> keySerde, - final Preconfigured> valueSerde) { - return new AutoMaterialized<>(keySerde, valueSerde, null, null, null, new HashMap<>(), true, true); - } - - /** - * @see Materialized#as(String) - */ - public static AutoMaterialized as(final String storeName) { - return new AutoMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), storeName, null, - null, new HashMap<>(), true, true); - } - - /** - * @see Materialized#as(DslStoreSuppliers) - */ - public static AutoMaterialized as( - final DslStoreSuppliers storeSuppliers) { - return new AutoMaterialized<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, - storeSuppliers, new HashMap<>(), true, true); - } - - Materialized configure(final Configurator configurator) { - final Materialized materialized = - this.storeName == null ? Materialized.with(configurator.configureForKeys(this.keySerde), - configurator.configureForValues(this.valueSerde)) : Materialized.as(this.storeName) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)); - if (this.retention != null) { - materialized.withRetention(this.retention); - } - if (this.storeType != null) { - materialized.withStoreType(this.storeType); - } - if (this.loggingEnabled) { - materialized.withLoggingEnabled(this.topicConfig); - } else { - materialized.withLoggingDisabled(); - } - if (this.cachingEnabled) { - materialized.withCachingEnabled(); - } else { - materialized.withCachingDisabled(); - } - return materialized; - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java deleted file mode 100644 index d98bb710..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoProduced.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.kstream.Produced; -import org.apache.kafka.streams.processor.StreamPartitioner; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Produced} using {@link Configurator} - * @param type of keys - * @param type of values - * @see Produced - */ -@With -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoProduced { - - private final @NonNull Preconfigured> keySerde; - private final @NonNull Preconfigured> valueSerde; - private final StreamPartitioner streamPartitioner; - private final String name; - - /** - * @see Produced#keySerde(Serde) - */ - public static AutoProduced keySerde(final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde()); - } - - /** - * @see Produced#valueSerde(Serde) - */ - public static AutoProduced valueSerde(final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see Produced#with(Serde, Serde) - */ - public static AutoProduced with(final Preconfigured> keySerde, - final Preconfigured> valueSerde) { - return new AutoProduced<>(keySerde, valueSerde, null, null); - } - - /** - * @see Produced#as(String) - */ - public static AutoProduced as(final String processorName) { - return new AutoProduced<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, - processorName); - } - - Produced configure(final Configurator configurator) { - return Produced.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) - .withStreamPartitioner(this.streamPartitioner); - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java deleted file mode 100644 index a63db629..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoRepartitioned.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.kstream.Repartitioned; -import org.apache.kafka.streams.processor.StreamPartitioner; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Repartitioned} using {@link Configurator} - * @param type of keys - * @param type of values - * @see Repartitioned - */ -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoRepartitioned { - - @With - private final @NonNull Preconfigured> keySerde; - @With - private final @NonNull Preconfigured> valueSerde; - @With - private final StreamPartitioner streamPartitioner; - @With - private final String name; - private final Integer numberOfPartitions; - - /** - * Create an instance of {@code AutoRepartitioned} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code AutoRepartitioned} - * @param type of keys - * @param type of values - */ - public static AutoRepartitioned keySerde(final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde()); - } - - /** - * Create an instance of {@code AutoRepartitioned} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code AutoRepartitioned} - * @param type of keys - * @param type of values - */ - public static AutoRepartitioned valueSerde(final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see Repartitioned#with(Serde, Serde) - */ - public static AutoRepartitioned with(final Preconfigured> keySerde, - final Preconfigured> valueSerde) { - return new AutoRepartitioned<>(keySerde, valueSerde, null, null, null); - } - - /** - * @see Repartitioned#as(String) - */ - public static AutoRepartitioned as(final String name) { - return new AutoRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, name, - null); - } - - /** - * @see Repartitioned#numberOfPartitions(int) - */ - public static AutoRepartitioned numberOfPartitions(final int numberOfPartitions) { - return new AutoRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), null, null, - numberOfPartitions); - } - - /** - * @see Repartitioned#streamPartitioner(StreamPartitioner) - */ - public static AutoRepartitioned streamPartitioner(final StreamPartitioner partitioner) { - return new AutoRepartitioned<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), partitioner, - null, null); - } - - /** - * @see Repartitioned#withNumberOfPartitions(int) - */ - public AutoRepartitioned withNumberOfPartitions(final int numberOfPartitions) { - return new AutoRepartitioned<>(this.keySerde, this.valueSerde, this.streamPartitioner, this.name, - numberOfPartitions); - } - - Repartitioned configure(final Configurator configurator) { - final Repartitioned repartitioned = Repartitioned.as(this.name) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) - .withStreamPartitioner(this.streamPartitioner); - return this.numberOfPartitions == null ? repartitioned : repartitioned - .withNumberOfPartitions(this.numberOfPartitions); - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java deleted file mode 100644 index 84de3992..00000000 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStreamJoined.java +++ /dev/null @@ -1,144 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2025 bakdata - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package com.bakdata.kafka; - -import java.util.HashMap; -import java.util.Map; -import lombok.AccessLevel; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.With; -import org.apache.kafka.common.serialization.Serde; -import org.apache.kafka.streams.kstream.StreamJoined; -import org.apache.kafka.streams.state.DslStoreSuppliers; - -/** - * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link StreamJoined} using {@link Configurator} - * @param type of keys - * @param this value type - * @param other value type - * @see StreamJoined - */ -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public final class AutoStreamJoined { - - @With - private final @NonNull Preconfigured> keySerde; - @With - private final @NonNull Preconfigured> valueSerde; - @With - private final @NonNull Preconfigured> otherValueSerde; - @With - private final DslStoreSuppliers dslStoreSuppliers; - @With - private final String name; - @With - private final String storeName; - private final Map topicConfig; - private final boolean loggingEnabled; - - /** - * Create an instance of {@code AutoStreamJoined} with provided key serde - * @param keySerde Serde to use for keys - * @return a new instance of {@code AutoStreamJoined} - * @param type of keys - * @param this value type - * @param other value type - */ - public static AutoStreamJoined keySerde( - final Preconfigured> keySerde) { - return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); - } - - /** - * Create an instance of {@code AutoStreamJoined} with provided value serde - * @param valueSerde Serde to use for values - * @return a new instance of {@code AutoStreamJoined} - * @param type of keys - * @param this value type - * @param other value type - */ - public static AutoStreamJoined valueSerde( - final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); - } - - /** - * Create an instance of {@code AutoStreamJoined} with provided other value serde - * @param valueSerde Serde to use for other values - * @return a new instance of {@code AutoStreamJoined} - * @param type of keys - * @param this value type - * @param other value type - */ - public static AutoStreamJoined otherValueSerde( - final Preconfigured> valueSerde) { - return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), valueSerde); - } - - /** - * @see StreamJoined#with(Serde, Serde, Serde) - */ - public static AutoStreamJoined with( - final Preconfigured> keySerde, - final Preconfigured> valueSerde, - final Preconfigured> otherValueSerde) { - return new AutoStreamJoined<>(keySerde, valueSerde, otherValueSerde, null, null, null, new HashMap<>(), - true); - } - - /** - * @see StreamJoined#as(String) - */ - public static AutoStreamJoined as(final String storeName) { - return new AutoStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde(), - null, null, storeName, new HashMap<>(), true); - } - - /** - * @see StreamJoined#with(DslStoreSuppliers) - */ - public static AutoStreamJoined with(final DslStoreSuppliers storeSuppliers) { - return new AutoStreamJoined<>(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), - Preconfigured.defaultSerde(), storeSuppliers, - null, null, new HashMap<>(), true); - } - - StreamJoined configure(final Configurator configurator) { - final StreamJoined streamJoined = StreamJoined.as(this.storeName) - .withKeySerde(configurator.configureForKeys(this.keySerde)) - .withValueSerde(configurator.configureForValues(this.valueSerde)) - .withOtherValueSerde(configurator.configureForValues(this.otherValueSerde)) - .withName(this.name) - .withDslStoreSuppliers(this.dslStoreSuppliers); - if (this.loggingEnabled) { - return streamJoined.withLoggingEnabled(this.topicConfig); - } else { - return streamJoined.withLoggingDisabled(); - } - } - -} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java index 8878fc20..701301aa 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamX.java @@ -50,7 +50,7 @@ public interface BranchedKStreamX extends BranchedKStream { BranchedKStreamX branch(Predicate predicate, BranchedX branched); /** - * @deprecated Use {@link #defaultBranchX()} + * @deprecated Use {@link #defaultBranchX()} instead. */ @Deprecated(since = "3.6.0") @Override @@ -62,7 +62,7 @@ public interface BranchedKStreamX extends BranchedKStream { Map> defaultBranchX(); /** - * @deprecated Use {@link #defaultBranchX(Branched)} + * @deprecated Use {@link #defaultBranchX(Branched)} instead. */ @Deprecated(since = "3.6.0") @Override @@ -79,7 +79,7 @@ public interface BranchedKStreamX extends BranchedKStream { Map> defaultBranch(BranchedX branched); /** - * @deprecated Use {@link #noDefaultBranchX()} + * @deprecated Use {@link #noDefaultBranchX()} instead. */ @Deprecated(since = "3.6.0") @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java index 0a89b6a1..742f6621 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedKStreamXImpl.java @@ -63,7 +63,7 @@ public BranchedKStreamX branch(final Predicate predi @Override public BranchedKStreamX branch(final Predicate predicate, final BranchedX branched) { - return this.branch(predicate, branched.convert(this.context)); + return this.branch(predicate, branched.configure(this.context)); } @Override @@ -88,7 +88,7 @@ public Map> defaultBranchX(final Branched branched) @Override public Map> defaultBranch(final BranchedX branched) { - return this.wrapX(this.defaultBranchInternal(branched.convert(this.context))); + return this.wrapX(this.defaultBranchInternal(branched.configure(this.context))); } @Override diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java index f2075547..a65dea32 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/BranchedX.java @@ -26,10 +26,6 @@ import java.util.function.Consumer; import java.util.function.Function; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; import org.apache.kafka.streams.kstream.Branched; import org.apache.kafka.streams.kstream.KStream; @@ -40,24 +36,17 @@ * @param type of keys * @param type of values */ -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public abstract class BranchedX { +public final class BranchedX extends ModifierChain, StreamsContext, BranchedX> { - @Getter(AccessLevel.PROTECTED) - private final String name; + private BranchedX(final Function> initializer) { + super(initializer); + } /** * @see Branched#withConsumer(Consumer) */ public static BranchedX withConsumer(final Consumer> chain) { - return new ConsumerBranchedX<>(null, chain); - } - - /** - * @see Branched#withConsumer(Consumer, String) - */ - public static BranchedX withConsumer(final Consumer> chain, final String name) { - return new ConsumerBranchedX<>(name, chain); + return new BranchedX<>(context -> Branched.withConsumer(asConsumer(chain, context))); } /** @@ -65,54 +54,35 @@ public static BranchedX withConsumer(final Consumer BranchedX withFunction( final Function, ? extends KStream> chain) { - return new FunctionBranchedX<>(null, chain); + return new BranchedX<>(context -> Branched.withFunction(asFunction(chain, context))); } /** - * @see Branched#withFunction(Function, String) + * @see Branched#as(String) */ - public static BranchedX withFunction( - final Function, ? extends KStream> chain, final String name) { - return new FunctionBranchedX<>(name, chain); + public static BranchedX as(final String name) { + return new BranchedX<>(context -> Branched.as(name)); } - abstract Branched convert(StreamsContext context); - - private static final class ConsumerBranchedX extends BranchedX { - private final @NonNull Consumer> chain; - - private ConsumerBranchedX(final String name, final Consumer> chain) { - super(name); - this.chain = chain; - } - - @Override - Branched convert(final StreamsContext context) { - return Branched.withConsumer(this.asConsumer(context), this.getName()); - } - - private Consumer> asConsumer(final StreamsContext context) { - return stream -> this.chain.accept(context.wrap(stream)); - } + private static Consumer> asConsumer(final Consumer> chain, + final StreamsContext context) { + return stream -> chain.accept(context.wrap(stream)); } - private static final class FunctionBranchedX extends BranchedX { - private final @NonNull Function, ? extends KStream> chain; - - private FunctionBranchedX(final String name, - final Function, ? extends KStream> chain) { - super(name); - this.chain = chain; - } - - @Override - Branched convert(final StreamsContext context) { - return Branched.withFunction(this.asFunction(context), this.getName()); - } + private static Function, KStream> asFunction( + final Function, ? extends KStream> chain, final StreamsContext context) { + return stream -> chain.apply(context.wrap(stream)); + } - private Function, KStream> asFunction(final StreamsContext context) { - return stream -> this.chain.apply(context.wrap(stream)); - } + /** + * @see Branched#withName(String) + */ + public BranchedX withName(final String name) { + return this.modify(branched -> branched.withName(name)); } + @Override + protected BranchedX newInstance(final Function> initializer) { + return new BranchedX<>(initializer); + } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java index a353d9df..f1fd5139 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedKStreamX.java @@ -63,7 +63,7 @@ KTableX aggregate(Initializer initializer, * @see #aggregate(Initializer, Materialized) */ KTableX aggregate(Initializer initializer, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX aggregate(Initializer initializer, Named named, @@ -73,7 +73,7 @@ KTableX aggregate(Initializer initializer, Named named, * @see #aggregate(Initializer, Named, Materialized) */ KTableX aggregate(Initializer initializer, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override TimeWindowedCogroupedKStreamX windowedBy(Windows windows); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java index 055dfa17..e5cf3251 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/CogroupedStreamXImpl.java @@ -70,7 +70,7 @@ public KTableX aggregate(final Initializer initializer, @Override public KTableX aggregate(final Initializer initializer, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); } @@ -82,7 +82,7 @@ public KTableX aggregate(final Initializer initializer, final Named nam @Override public KTableX aggregate(final Initializer initializer, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java new file mode 100644 index 00000000..e530ba3e --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ConsumedX.java @@ -0,0 +1,181 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.kstream.Consumed; +import org.apache.kafka.streams.processor.TimestampExtractor; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Consumed} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Consumed + */ +public final class ConsumedX extends ModifierChain, Configurator, ConsumedX> { + + private ConsumedX(final Function> initializer) { + super(initializer); + } + + /** + * Create an instance of {@code ConsumedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConsumedX} + * @param type of keys + * @param type of values + */ + public static ConsumedX keySerde(final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code ConsumedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code ConsumedX} + * @param type of keys + * @param type of values + */ + public static ConsumedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * Create an instance of {@code ConsumedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConsumedX} + * @param type of keys + * @param type of values + */ + public static ConsumedX valueSerde(final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde); + } + + /** + * Create an instance of {@code ConsumedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code ConsumedX} + * @param type of keys + * @param type of values + */ + public static ConsumedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Consumed#with(Serde, Serde) + */ + public static ConsumedX with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ConsumedX<>(configurator -> Consumed.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde))); + } + + /** + * @see Consumed#with(Serde, Serde) + */ + public static ConsumedX with(final Serde keySerde, final Serde valueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + + /** + * @see Consumed#with(TimestampExtractor) + */ + public static ConsumedX with(final TimestampExtractor timestampExtractor) { + return new ConsumedX<>(configurator -> Consumed.with(timestampExtractor)); + } + + /** + * @see Consumed#with(AutoOffsetReset) + */ + public static ConsumedX with(final AutoOffsetReset resetPolicy) { + return new ConsumedX<>(configurator -> Consumed.with(resetPolicy)); + } + + /** + * @see Consumed#as(String) + */ + public static ConsumedX as(final String processorName) { + return new ConsumedX<>(configurator -> Consumed.as(processorName)); + } + + /** + * @see Consumed#withKeySerde(Serde) + */ + public ConsumedX withKeySerde(final Preconfigured> keySerde) { + return this.modify((consumed, configurator) -> consumed.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Consumed#withKeySerde(Serde) + */ + public ConsumedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Consumed#withValueSerde(Serde) + */ + public ConsumedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify( + (consumed, configurator) -> consumed.withValueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Consumed#withValueSerde(Serde) + */ + public ConsumedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Consumed#withOffsetResetPolicy(AutoOffsetReset) + */ + public ConsumedX withOffsetResetPolicy(final AutoOffsetReset offsetResetPolicy) { + return this.modify(consumed -> consumed.withOffsetResetPolicy(offsetResetPolicy)); + } + + /** + * @see Consumed#withTimestampExtractor(TimestampExtractor) + */ + public ConsumedX withTimestampExtractor(final TimestampExtractor timestampExtractor) { + return this.modify(consumed -> consumed.withTimestampExtractor(timestampExtractor)); + } + + /** + * @see Consumed#withName(String) + */ + public ConsumedX withName(final String processorName) { + return this.modify(consumed -> consumed.withName(processorName)); + } + + @Override + protected ConsumedX newInstance(final Function> initializer) { + return new ConsumedX<>(initializer); + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/GroupedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/GroupedX.java new file mode 100644 index 00000000..979742ea --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/GroupedX.java @@ -0,0 +1,135 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Grouped; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Grouped} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Grouped + */ +public final class GroupedX extends ModifierChain, Configurator, GroupedX> { + + private GroupedX(final Function> initializer) { + super(initializer); + } + + /** + * @see Grouped#keySerde(Serde) + */ + public static GroupedX keySerde(final Preconfigured> keySerde) { + return new GroupedX<>(configurator -> Grouped.keySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Grouped#keySerde(Serde) + */ + public static GroupedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Grouped#valueSerde(Serde) + */ + public static GroupedX valueSerde(final Preconfigured> valueSerde) { + return new GroupedX<>(configurator -> Grouped.valueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Grouped#valueSerde(Serde) + */ + public static GroupedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Grouped#with(Serde, Serde) + */ + public static GroupedX with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new GroupedX<>(configurator -> Grouped.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde))); + } + + /** + * @see Grouped#with(Serde, Serde) + */ + public static GroupedX with(final Serde keySerde, final Serde valueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + + /** + * @see Grouped#as(String) + */ + public static GroupedX as(final String name) { + return new GroupedX<>(configurator -> Grouped.as(name)); + } + + /** + * @see Grouped#withKeySerde(Serde) + */ + public GroupedX withKeySerde(final Preconfigured> keySerde) { + return this.modify((grouped, configurator) -> grouped.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Grouped#withKeySerde(Serde) + */ + public GroupedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Grouped#withValueSerde(Serde) + */ + public GroupedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify( + (grouped, configurator) -> grouped.withValueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Grouped#withValueSerde(Serde) + */ + public GroupedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Grouped#withName(String) + */ + public GroupedX withName(final String processorName) { + return this.modify(grouped -> grouped.withName(processorName)); + } + + @Override + protected GroupedX newInstance(final Function> initializer) { + return new GroupedX<>(initializer); + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/JoinedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/JoinedX.java new file mode 100644 index 00000000..36957f02 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/JoinedX.java @@ -0,0 +1,179 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.time.Duration; +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Joined; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Joined} using {@link Configurator} + * @param type of keys + * @param this value type + * @param other value type + * @see Joined + */ +public final class JoinedX extends ModifierChain, Configurator, JoinedX> { + + private JoinedX(final Function> initializer) { + super(initializer); + } + + /** + * @see Joined#keySerde(Serde) + */ + public static JoinedX keySerde(final Preconfigured> keySerde) { + return new JoinedX<>(configurator -> Joined.keySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Joined#keySerde(Serde) + */ + public static JoinedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Joined#valueSerde(Serde) + */ + public static JoinedX valueSerde(final Preconfigured> valueSerde) { + return new JoinedX<>(configurator -> Joined.valueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Joined#valueSerde(Serde) + */ + public static JoinedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Joined#otherValueSerde(Serde) + */ + public static JoinedX otherValueSerde( + final Preconfigured> otherValueSerde) { + return new JoinedX<>(configurator -> Joined.otherValueSerde(configurator.configureForValues(otherValueSerde))); + } + + /** + * @see Joined#otherValueSerde(Serde) + */ + public static JoinedX otherValueSerde(final Serde otherValueSerde) { + return otherValueSerde(Preconfigured.create(otherValueSerde)); + } + + /** + * @see Joined#with(Serde, Serde, Serde) + */ + public static JoinedX with( + final Preconfigured> keySerde, + final Preconfigured> valueSerde, + final Preconfigured> otherValueSerde) { + return new JoinedX<>(configurator -> Joined.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde), configurator.configureForValues(otherValueSerde))); + } + + /** + * @see Joined#with(Serde, Serde, Serde) + */ + public static JoinedX with( + final Serde keySerde, + final Serde valueSerde, + final Serde otherValueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde), + Preconfigured.create(otherValueSerde)); + } + + /** + * @see Joined#as(String) + */ + public static JoinedX as(final String name) { + return new JoinedX<>(configurator -> Joined.as(name)); + } + + /** + * @see Joined#withKeySerde(Serde) + */ + public JoinedX withKeySerde(final Preconfigured> keySerde) { + return this.modify((joined, configurator) -> joined.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Joined#withKeySerde(Serde) + */ + public JoinedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Joined#withValueSerde(Serde) + */ + public JoinedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify( + (joined, configurator) -> joined.withValueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Joined#withValueSerde(Serde) + */ + public JoinedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Joined#withOtherValueSerde(Serde) + */ + public JoinedX withOtherValueSerde(final Preconfigured> otherValueSerde) { + return this.modify( + (joined, configurator) -> joined.withOtherValueSerde(configurator.configureForValues(otherValueSerde))); + } + + /** + * @see Joined#withOtherValueSerde(Serde) + */ + public JoinedX withOtherValueSerde(final Serde otherValueSerde) { + return this.withOtherValueSerde(Preconfigured.create(otherValueSerde)); + } + + /** + * @see Joined#withName(String) + */ + public JoinedX withName(final String name) { + return this.modify(joined -> joined.withName(name)); + } + + /** + * @see Joined#withGracePeriod(Duration) + */ + public JoinedX withGracePeriod(final Duration gracePeriod) { + return this.modify(joined -> joined.withGracePeriod(gracePeriod)); + } + + @Override + protected JoinedX newInstance(final Function> initializer) { + return new JoinedX<>(initializer); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java index 3d697a3c..638c7cd6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamX.java @@ -57,7 +57,7 @@ public interface KGroupedStreamX extends KGroupedStream { /** * @see #count(Materialized) */ - KTableX count(AutoMaterialized> materialized); + KTableX count(MaterializedX> materialized); @Override KTableX count(Named named, Materialized> materialized); @@ -66,7 +66,7 @@ public interface KGroupedStreamX extends KGroupedStream { * @see #count(Named, Materialized) */ KTableX count(Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX reduce(Reducer reducer); @@ -78,7 +78,7 @@ KTableX count(Named named, * @see #reduce(Reducer, Materialized) */ KTableX reduce(Reducer reducer, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX reduce(Reducer reducer, Named named, @@ -88,7 +88,7 @@ KTableX reduce(Reducer reducer, Named named, * @see #reduce(Reducer, Named, Materialized) */ KTableX reduce(Reducer reducer, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX aggregate(Initializer initializer, Aggregator aggregator); @@ -101,7 +101,7 @@ KTableX aggregate(Initializer initializer, Aggregator KTableX aggregate(Initializer initializer, Aggregator aggregator, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX aggregate(Initializer initializer, Aggregator aggregator, @@ -111,7 +111,7 @@ KTableX aggregate(Initializer initializer, Aggregator KTableX aggregate(Initializer initializer, Aggregator aggregator, - Named named, AutoMaterialized> materialized); + Named named, MaterializedX> materialized); @Override TimeWindowedKStreamX windowedBy(Windows windows); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java index da954f02..8c45499c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedStreamXImpl.java @@ -65,7 +65,7 @@ public KTableX count(final Materialized count( - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -77,7 +77,7 @@ public KTableX count(final Named named, @Override public KTableX count(final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -94,7 +94,7 @@ public KTableX reduce(final Reducer reducer, @Override public KTableX reduce(final Reducer reducer, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @@ -106,7 +106,7 @@ public KTableX reduce(final Reducer reducer, final Named named, @Override public KTableX reduce(final Reducer reducer, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } @@ -126,7 +126,7 @@ public KTableX aggregate(final Initializer initializer, @Override public KTableX aggregate(final Initializer initializer, final Aggregator aggregator, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); } @@ -140,7 +140,7 @@ public KTableX aggregate(final Initializer initializer, @Override public KTableX aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java index e59b3764..7594cf14 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableX.java @@ -47,7 +47,7 @@ public interface KGroupedTableX extends KGroupedTable { /** * @see #count(Materialized) */ - KTableX count(AutoMaterialized> materialized); + KTableX count(MaterializedX> materialized); @Override KTableX count(Named named, Materialized> materialized); @@ -56,7 +56,7 @@ public interface KGroupedTableX extends KGroupedTable { * @see #count(Named, Materialized) */ KTableX count(Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX count(); @@ -72,7 +72,7 @@ KTableX reduce(Reducer adder, Reducer subtractor, * @see #reduce(Reducer, Reducer, Materialized) */ KTableX reduce(Reducer adder, Reducer subtractor, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX reduce(Reducer adder, Reducer subtractor, Named named, @@ -82,7 +82,7 @@ KTableX reduce(Reducer adder, Reducer subtractor, Named named, * @see #reduce(Reducer, Reducer, Named, Materialized) */ KTableX reduce(Reducer adder, Reducer subtractor, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX reduce(Reducer adder, Reducer subtractor); @@ -97,7 +97,7 @@ KTableX aggregate(Initializer initializer, Aggregator KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX aggregate(Initializer initializer, Aggregator adder, @@ -109,7 +109,7 @@ KTableX aggregate(Initializer initializer, Aggregator KTableX aggregate(Initializer initializer, Aggregator adder, Aggregator subtractor, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX aggregate(Initializer initializer, Aggregator adder, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java index ab40e603..a875aeb7 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KGroupedTableXImpl.java @@ -48,7 +48,7 @@ public KTableX count(final Materialized count( - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -60,7 +60,7 @@ public KTableX count(final Named named, @Override public KTableX count(final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -82,7 +82,7 @@ public KTableX reduce(final Reducer adder, final Reducer subtractor, @Override public KTableX reduce(final Reducer adder, final Reducer subtractor, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(adder, subtractor, materialized.configure(this.context.getConfigurator())); } @@ -94,7 +94,7 @@ public KTableX reduce(final Reducer adder, final Reducer subtractor, @Override public KTableX reduce(final Reducer adder, final Reducer subtractor, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(adder, subtractor, named, materialized.configure(this.context.getConfigurator())); } @@ -115,7 +115,7 @@ public KTableX aggregate(final Initializer initializer, public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, adder, subtractor, materialized.configure(this.context.getConfigurator())); } @@ -131,7 +131,7 @@ public KTableX aggregate(final Initializer initializer, public KTableX aggregate(final Initializer initializer, final Aggregator adder, final Aggregator subtractor, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, adder, subtractor, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java index 689eae23..6a01db0d 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamX.java @@ -479,9 +479,11 @@ KErrorStream flatMapValuesCapturingErrors( @Override KStreamX peek(ForeachAction action, Named named); + @Deprecated @Override KStreamX[] branch(Named named, Predicate... predicates); + @Deprecated @Override KStreamX[] branch(Predicate... predicates); @@ -497,9 +499,11 @@ KErrorStream flatMapValuesCapturingErrors( @Override KStreamX merge(KStream stream, Named named); + @Deprecated @Override KStreamX through(String topic); + @Deprecated @Override KStreamX through(String topic, Produced produced); @@ -512,17 +516,17 @@ KErrorStream flatMapValuesCapturingErrors( /** * @see #repartition(Repartitioned) */ - KStreamX repartition(AutoRepartitioned repartitioned); + KStreamX repartition(RepartitionedX repartitioned); /** * @see #to(String, Produced) */ - void to(String topic, AutoProduced produced); + void to(String topic, ProducedX produced); /** * @see #to(TopicNameExtractor, Produced) */ - void to(TopicNameExtractor topicExtractor, AutoProduced produced); + void to(TopicNameExtractor topicExtractor, ProducedX produced); /** * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic()} @@ -542,7 +546,7 @@ KErrorStream flatMapValuesCapturingErrors( * @param produced define optional parameters for materializing the stream * @see #to(String, Produced) */ - void toOutputTopic(AutoProduced produced); + void toOutputTopic(ProducedX produced); /** * Materialize {@link KStream} to {@link StreamsTopicConfig#getOutputTopic(String)} @@ -565,7 +569,7 @@ KErrorStream flatMapValuesCapturingErrors( * @param produced define optional parameters for materializing the stream * @see #to(String, Produced) */ - void toOutputTopic(String label, AutoProduced produced); + void toOutputTopic(String label, ProducedX produced); /** * Materialize {@link KStream} to {@link StreamsTopicConfig#getErrorTopic()} @@ -585,7 +589,7 @@ KErrorStream flatMapValuesCapturingErrors( * @param produced define optional parameters for materializing the stream * @see #to(String, Produced) */ - void toErrorTopic(AutoProduced produced); + void toErrorTopic(ProducedX produced); @Override KTableX toTable(); @@ -599,7 +603,7 @@ KErrorStream flatMapValuesCapturingErrors( /** * @see #toTable(Materialized) */ - KTableX toTable(AutoMaterialized> materialized); + KTableX toTable(MaterializedX> materialized); @Override KTableX toTable(Named named, Materialized> materialized); @@ -607,7 +611,7 @@ KErrorStream flatMapValuesCapturingErrors( /** * @see #toTable(Named, Materialized) */ - KTableX toTable(Named named, AutoMaterialized> materialized); + KTableX toTable(Named named, MaterializedX> materialized); @Override KGroupedStreamX groupBy(KeyValueMapper keySelector); @@ -620,7 +624,7 @@ KGroupedStreamX groupBy(KeyValueMapper key * @see #groupBy(KeyValueMapper, Grouped) */ KGroupedStreamX groupBy(KeyValueMapper keySelector, - AutoGrouped grouped); + GroupedX grouped); @Override KGroupedStreamX groupByKey(); @@ -631,7 +635,7 @@ KGroupedStreamX groupBy(KeyValueMapper key /** * @see #groupByKey(Grouped) */ - KGroupedStreamX groupByKey(AutoGrouped grouped); + KGroupedStreamX groupByKey(GroupedX grouped); @Override KStreamX join(KStream otherStream, @@ -652,7 +656,7 @@ KStreamX join(KStream otherStream, */ KStreamX join(KStream otherStream, ValueJoiner joiner, - JoinWindows windows, AutoStreamJoined streamJoined); + JoinWindows windows, StreamJoinedX streamJoined); @Override KStreamX join(KStream otherStream, @@ -664,7 +668,7 @@ KStreamX join(KStream otherStream, */ KStreamX join(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, - AutoStreamJoined streamJoined); + StreamJoinedX streamJoined); @Override KStreamX leftJoin(KStream otherStream, @@ -684,7 +688,7 @@ KStreamX leftJoin(KStream otherStream, */ KStreamX leftJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, - AutoStreamJoined streamJoined); + StreamJoinedX streamJoined); @Override KStreamX leftJoin(KStream otherStream, @@ -696,7 +700,7 @@ KStreamX leftJoin(KStream otherStream, */ KStreamX leftJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, - AutoStreamJoined streamJoined); + StreamJoinedX streamJoined); @Override KStreamX outerJoin(KStream otherStream, @@ -716,7 +720,7 @@ KStreamX outerJoin(KStream otherStream, */ KStreamX outerJoin(KStream otherStream, ValueJoiner joiner, JoinWindows windows, - AutoStreamJoined streamJoined); + StreamJoinedX streamJoined); @Override KStreamX outerJoin(KStream otherStream, @@ -728,7 +732,7 @@ KStreamX outerJoin(KStream otherStream, */ KStreamX outerJoin(KStream otherStream, ValueJoinerWithKey joiner, JoinWindows windows, - AutoStreamJoined streamJoined); + StreamJoinedX streamJoined); @Override KStreamX join(KTable table, ValueJoiner joiner); @@ -745,7 +749,7 @@ KStreamX join(KTable table, ValueJoiner KStreamX join(KTable table, ValueJoiner joiner, - AutoJoined joined); + JoinedX joined); @Override KStreamX join(KTable table, @@ -756,7 +760,7 @@ KStreamX join(KTable table, */ KStreamX join(KTable table, ValueJoinerWithKey joiner, - AutoJoined joined); + JoinedX joined); @Override KStreamX leftJoin(KTable table, @@ -776,7 +780,7 @@ KStreamX leftJoin(KTable table, */ KStreamX leftJoin(KTable table, ValueJoiner joiner, - AutoJoined joined); + JoinedX joined); @Override KStreamX leftJoin(KTable table, @@ -787,7 +791,7 @@ KStreamX leftJoin(KTable table, */ KStreamX leftJoin(KTable table, ValueJoinerWithKey joiner, - AutoJoined joined); + JoinedX joined); @Override KStreamX join(GlobalKTable globalTable, @@ -829,61 +833,73 @@ KStreamX leftJoin(GlobalKTable globalTable, KeyValueMapper keySelector, ValueJoinerWithKey valueJoiner, Named named); + @Deprecated @Override KStreamX transform( TransformerSupplier> transformerSupplier, String... stateStoreNames); + @Deprecated @Override KStreamX transform( TransformerSupplier> transformerSupplier, Named named, String... stateStoreNames); + @Deprecated @Override KStreamX flatTransform( TransformerSupplier>> transformerSupplier, String... stateStoreNames); + @Deprecated @Override KStreamX flatTransform( TransformerSupplier>> transformerSupplier, Named named, String... stateStoreNames); + @Deprecated @Override KStreamX transformValues( ValueTransformerSupplier valueTransformerSupplier, String... stateStoreNames); + @Deprecated @Override KStreamX transformValues( ValueTransformerSupplier valueTransformerSupplier, Named named, String... stateStoreNames); + @Deprecated @Override KStreamX transformValues( ValueTransformerWithKeySupplier valueTransformerSupplier, String... stateStoreNames); + @Deprecated @Override KStreamX transformValues( ValueTransformerWithKeySupplier valueTransformerSupplier, Named named, String... stateStoreNames); + @Deprecated @Override KStreamX flatTransformValues( ValueTransformerSupplier> valueTransformerSupplier, String... stateStoreNames); + @Deprecated @Override KStreamX flatTransformValues( ValueTransformerSupplier> valueTransformerSupplier, Named named, String... stateStoreNames); + @Deprecated @Override KStreamX flatTransformValues( ValueTransformerWithKeySupplier> valueTransformerSupplier, String... stateStoreNames); + @Deprecated @Override KStreamX flatTransformValues( ValueTransformerWithKeySupplier> valueTransformerSupplier, Named named, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java index f51685a6..88e99aaf 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KStreamXImpl.java @@ -426,7 +426,7 @@ public KStreamX repartition(final Repartitioned repartitioned) { } @Override - public KStreamX repartition(final AutoRepartitioned repartitioned) { + public KStreamX repartition(final RepartitionedX repartitioned) { return this.repartition(repartitioned.configure(this.context.getConfigurator())); } @@ -441,7 +441,7 @@ public void to(final String topic, final Produced produced) { } @Override - public void to(final String topic, final AutoProduced produced) { + public void to(final String topic, final ProducedX produced) { this.to(topic, produced.configure(this.context.getConfigurator())); } @@ -456,7 +456,7 @@ public void to(final TopicNameExtractor topicExtractor, final Produced topicExtractor, final AutoProduced produced) { + public void to(final TopicNameExtractor topicExtractor, final ProducedX produced) { this.to(topicExtractor, produced.configure(this.context.getConfigurator())); } @@ -471,7 +471,7 @@ public void toOutputTopic(final Produced produced) { } @Override - public void toOutputTopic(final AutoProduced produced) { + public void toOutputTopic(final ProducedX produced) { this.toOutputTopic(produced.configure(this.context.getConfigurator())); } @@ -486,7 +486,7 @@ public void toOutputTopic(final String label, final Produced produced) { } @Override - public void toOutputTopic(final String label, final AutoProduced produced) { + public void toOutputTopic(final String label, final ProducedX produced) { this.toOutputTopic(label, produced.configure(this.context.getConfigurator())); } @@ -501,7 +501,7 @@ public void toErrorTopic(final Produced produced) { } @Override - public void toErrorTopic(final AutoProduced produced) { + public void toErrorTopic(final ProducedX produced) { this.toErrorTopic(produced.configure(this.context.getConfigurator())); } @@ -521,7 +521,7 @@ public KTableX toTable(final Materialized toTable(final AutoMaterialized> materialized) { + public KTableX toTable(final MaterializedX> materialized) { return this.toTable(materialized.configure(this.context.getConfigurator())); } @@ -533,7 +533,7 @@ public KTableX toTable(final Named named, @Override public KTableX toTable(final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.toTable(named, materialized.configure(this.context.getConfigurator())); } @@ -550,7 +550,7 @@ public KGroupedStreamX groupBy(final KeyValueMapper KGroupedStreamX groupBy(final KeyValueMapper keySelector, - final AutoGrouped grouped) { + final GroupedX grouped) { return this.groupBy(keySelector, grouped.configure(this.context.getConfigurator())); } @@ -565,7 +565,7 @@ public KGroupedStreamX groupByKey(final Grouped grouped) { } @Override - public KGroupedStreamX groupByKey(final AutoGrouped grouped) { + public KGroupedStreamX groupByKey(final GroupedX grouped) { return this.groupByKey(grouped.configure(this.context.getConfigurator())); } @@ -595,7 +595,7 @@ public KStreamX join(final KStream otherStream, @Override public KStreamX join(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final AutoStreamJoined streamJoined) { + final StreamJoinedX streamJoined) { return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -610,7 +610,7 @@ public KStreamX join(final KStream otherStream, @Override public KStreamX join(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, - final AutoStreamJoined streamJoined) { + final StreamJoinedX streamJoined) { return this.join(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -640,7 +640,7 @@ public KStreamX leftJoin(final KStream otherStream, @Override public KStreamX leftJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final AutoStreamJoined streamJoined) { + final StreamJoinedX streamJoined) { return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -655,7 +655,7 @@ public KStreamX leftJoin(final KStream otherStream, @Override public KStreamX leftJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, - final AutoStreamJoined streamJoined) { + final StreamJoinedX streamJoined) { return this.leftJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -685,7 +685,7 @@ public KStreamX outerJoin(final KStream otherStream, @Override public KStreamX outerJoin(final KStream otherStream, final ValueJoiner joiner, final JoinWindows windows, - final AutoStreamJoined streamJoined) { + final StreamJoinedX streamJoined) { return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -700,7 +700,7 @@ public KStreamX outerJoin(final KStream otherStream, @Override public KStreamX outerJoin(final KStream otherStream, final ValueJoinerWithKey joiner, final JoinWindows windows, - final AutoStreamJoined streamJoined) { + final StreamJoinedX streamJoined) { return this.outerJoin(otherStream, joiner, windows, streamJoined.configure(this.context.getConfigurator())); } @@ -727,7 +727,7 @@ public KStreamX join(final KTable table, @Override public KStreamX join(final KTable table, - final ValueJoiner joiner, final AutoJoined joined) { + final ValueJoiner joiner, final JoinedX joined) { return this.join(table, joiner, joined.configure(this.context.getConfigurator())); } @@ -742,7 +742,7 @@ public KStreamX join(final KTable table, @Override public KStreamX join(final KTable table, final ValueJoinerWithKey joiner, - final AutoJoined joined) { + final JoinedX joined) { return this.join(table, joiner, joined.configure(this.context.getConfigurator())); } @@ -769,7 +769,7 @@ public KStreamX leftJoin(final KTable table, @Override public KStreamX leftJoin(final KTable table, - final ValueJoiner joiner, final AutoJoined joined) { + final ValueJoiner joiner, final JoinedX joined) { return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); } @@ -784,7 +784,7 @@ public KStreamX leftJoin(final KTable table, @Override public KStreamX leftJoin(final KTable table, final ValueJoinerWithKey joiner, - final AutoJoined joined) { + final JoinedX joined) { return this.leftJoin(table, joiner, joined.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index c7d4217d..42f42d26 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -63,7 +63,7 @@ KTableX filter(Predicate predicate, * @see #filter(Predicate, Materialized) */ KTableX filter(Predicate predicate, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX filter(Predicate predicate, Named named, @@ -73,7 +73,7 @@ KTableX filter(Predicate predicate, Named named, * @see #filter(Predicate, Named, Materialized) */ KTableX filter(Predicate predicate, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX filterNot(Predicate predicate); @@ -89,7 +89,7 @@ KTableX filterNot(Predicate predicate, * @see #filterNot(Predicate, Materialized) */ KTableX filterNot(Predicate predicate, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX filterNot(Predicate predicate, Named named, @@ -99,7 +99,7 @@ KTableX filterNot(Predicate predicate, Named named, * @see #filterNot(Predicate, Named, Materialized) */ KTableX filterNot(Predicate predicate, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX mapValues(ValueMapper mapper); @@ -121,7 +121,7 @@ KTableX mapValues(ValueMapper mapper, * @see #mapValues(ValueMapper, Materialized) */ KTableX mapValues(ValueMapper mapper, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX mapValues(ValueMapper mapper, Named named, @@ -131,7 +131,7 @@ KTableX mapValues(ValueMapper mapper, Named * @see #mapValues(ValueMapper, Named, Materialized) */ KTableX mapValues(ValueMapper mapper, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX mapValues(ValueMapperWithKey mapper, @@ -141,7 +141,7 @@ KTableX mapValues(ValueMapperWithKey KTableX mapValues(ValueMapperWithKey mapper, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX mapValues(ValueMapperWithKey mapper, Named named, @@ -151,7 +151,7 @@ KTableX mapValues(ValueMapperWithKey KTableX mapValues(ValueMapperWithKey mapper, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KStreamX toStream(); @@ -188,7 +188,7 @@ KTableX transformValues( */ KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, - AutoMaterialized> materialized, String... stateStoreNames); + MaterializedX> materialized, String... stateStoreNames); @Override KTableX transformValues( @@ -200,7 +200,7 @@ KTableX transformValues( */ KTableX transformValues( ValueTransformerWithKeySupplier transformerSupplier, - AutoMaterialized> materialized, Named named, + MaterializedX> materialized, Named named, String... stateStoreNames); @Override @@ -214,7 +214,7 @@ KGroupedTableX groupBy(KeyValueMapper KGroupedTableX groupBy(KeyValueMapper> selector, - AutoGrouped grouped); + GroupedX grouped); @Override KTableX join(KTable other, ValueJoiner joiner); @@ -231,7 +231,7 @@ KTableX join(KTable other, ValueJoiner KTableX join(KTable other, ValueJoiner joiner, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX join(KTable other, ValueJoiner joiner, @@ -241,7 +241,7 @@ KTableX join(KTable other, ValueJoiner KTableX join(KTable other, ValueJoiner joiner, - Named named, AutoMaterialized> materialized); + Named named, MaterializedX> materialized); @Override KTableX leftJoin(KTable other, @@ -262,7 +262,7 @@ KTableX leftJoin(KTable other, */ KTableX leftJoin(KTable other, ValueJoiner joiner, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX leftJoin(KTable other, @@ -274,7 +274,7 @@ KTableX leftJoin(KTable other, */ KTableX leftJoin(KTable other, ValueJoiner joiner, - Named named, AutoMaterialized> materialized); + Named named, MaterializedX> materialized); @Override KTableX outerJoin(KTable other, @@ -295,7 +295,7 @@ KTableX outerJoin(KTable other, */ KTableX outerJoin(KTable other, ValueJoiner joiner, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX outerJoin(KTable other, @@ -307,12 +307,13 @@ KTableX outerJoin(KTable other, */ KTableX outerJoin(KTable other, ValueJoiner joiner, - Named named, AutoMaterialized> materialized); + Named named, MaterializedX> materialized); @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); + @Deprecated @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named); @@ -329,8 +330,9 @@ KTableX join(KTable other, Function foreignKe * @see #join(KTable, Function, ValueJoiner, Materialized) */ KTableX join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, AutoMaterialized> materialized); + ValueJoiner joiner, MaterializedX> materialized); + @Deprecated @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); @@ -340,7 +342,7 @@ KTableX join(KTable other, Function foreignKe */ KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX join(KTable other, Function foreignKeyExtractor, @@ -352,12 +354,13 @@ KTableX join(KTable other, Function foreignKe */ KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner); + @Deprecated @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named); @@ -374,8 +377,9 @@ KTableX leftJoin(KTable other, Function forei * @see #leftJoin(KTable, Function, ValueJoiner, Materialized) */ KTableX leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, AutoMaterialized> materialized); + ValueJoiner joiner, MaterializedX> materialized); + @Deprecated @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); @@ -385,7 +389,7 @@ KTableX leftJoin(KTable other, Function forei */ KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, @@ -397,5 +401,5 @@ KTableX leftJoin(KTable other, Function forei */ KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, - AutoMaterialized> materialized); + MaterializedX> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java index 7063ed86..baec63ff 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java @@ -70,7 +70,7 @@ public KTableX filter(final Predicate predicate, @Override public KTableX filter(final Predicate predicate, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.filter(predicate, materialized.configure(this.context.getConfigurator())); } @@ -82,7 +82,7 @@ public KTableX filter(final Predicate predicate, fin @Override public KTableX filter(final Predicate predicate, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.filter(predicate, named, materialized.configure(this.context.getConfigurator())); } @@ -104,7 +104,7 @@ public KTableX filterNot(final Predicate predicate, @Override public KTableX filterNot(final Predicate predicate, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.filterNot(predicate, materialized.configure(this.context.getConfigurator())); } @@ -116,7 +116,7 @@ public KTableX filterNot(final Predicate predicate, @Override public KTableX filterNot(final Predicate predicate, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.filterNot(predicate, named, materialized.configure(this.context.getConfigurator())); } @@ -149,7 +149,7 @@ public KTableX mapValues(final ValueMapper @Override public KTableX mapValues(final ValueMapper mapper, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); } @@ -161,7 +161,7 @@ public KTableX mapValues(final ValueMapper @Override public KTableX mapValues(final ValueMapper mapper, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); } @@ -173,7 +173,7 @@ public KTableX mapValues(final ValueMapperWithKey KTableX mapValues(final ValueMapperWithKey mapper, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.mapValues(mapper, materialized.configure(this.context.getConfigurator())); } @@ -185,7 +185,7 @@ public KTableX mapValues(final ValueMapperWithKey KTableX mapValues(final ValueMapperWithKey mapper, - final Named named, final AutoMaterialized> materialized) { + final Named named, final MaterializedX> materialized) { return this.mapValues(mapper, named, materialized.configure(this.context.getConfigurator())); } @@ -240,7 +240,7 @@ public KTableX transformValues( @Override public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, - final AutoMaterialized> materialized, + final MaterializedX> materialized, final String... stateStoreNames) { return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), stateStoreNames); @@ -258,7 +258,7 @@ public KTableX transformValues( @Override public KTableX transformValues( final ValueTransformerWithKeySupplier transformerSupplier, - final AutoMaterialized> materialized, final Named named, + final MaterializedX> materialized, final Named named, final String... stateStoreNames) { return this.transformValues(transformerSupplier, materialized.configure(this.context.getConfigurator()), named, stateStoreNames); @@ -278,7 +278,7 @@ public KGroupedTableX groupBy( @Override public KGroupedTableX groupBy( - final KeyValueMapper> selector, final AutoGrouped grouped) { + final KeyValueMapper> selector, final GroupedX grouped) { return this.groupBy(selector, grouped.configure(this.context.getConfigurator())); } @@ -307,7 +307,7 @@ public KTableX join(final KTable other, @Override public KTableX join(final KTable other, final ValueJoiner joiner, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.join(other, joiner, materialized.configure(this.context.getConfigurator())); } @@ -322,7 +322,7 @@ public KTableX join(final KTable other, @Override public KTableX join(final KTable other, final ValueJoiner joiner, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.join(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -351,7 +351,7 @@ public KTableX leftJoin(final KTable other, @Override public KTableX leftJoin(final KTable other, final ValueJoiner joiner, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.leftJoin(other, joiner, materialized.configure(this.context.getConfigurator())); } @@ -366,7 +366,7 @@ public KTableX leftJoin(final KTable other, @Override public KTableX leftJoin(final KTable other, final ValueJoiner joiner, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.leftJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -395,7 +395,7 @@ public KTableX outerJoin(final KTable other, @Override public KTableX outerJoin(final KTable other, final ValueJoiner joiner, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.outerJoin(other, joiner, materialized.configure(this.context.getConfigurator())); } @@ -410,7 +410,7 @@ public KTableX outerJoin(final KTable other, @Override public KTableX outerJoin(final KTable other, final ValueJoiner joiner, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.outerJoin(other, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -450,7 +450,7 @@ public KTableX join(final KTable other, public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.join(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } @@ -467,7 +467,7 @@ public KTableX join(final KTable other, public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.join(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -485,7 +485,7 @@ public KTableX join(final KTable other, public KTableX join(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.join(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); } @@ -526,7 +526,7 @@ public KTableX leftJoin(final KTable other, public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, materialized.configure(this.context.getConfigurator())); } @@ -544,7 +544,7 @@ public KTableX leftJoin(final KTable other, public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, named, materialized.configure(this.context.getConfigurator())); } @@ -563,7 +563,7 @@ public KTableX leftJoin(final KTable other, public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, final ValueJoiner joiner, final TableJoined tableJoined, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.leftJoin(other, foreignKeyExtractor, joiner, tableJoined, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/MaterializedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/MaterializedX.java new file mode 100644 index 00000000..2fc98b6e --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/MaterializedX.java @@ -0,0 +1,238 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.time.Duration; +import java.util.Map; +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.state.DslStoreSuppliers; +import org.apache.kafka.streams.state.KeyValueBytesStoreSupplier; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.SessionBytesStoreSupplier; +import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.WindowBytesStoreSupplier; +import org.apache.kafka.streams.state.WindowStore; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Materialized} using {@link Configurator} + * @param type of keys + * @param type of values + * @param type of state store + * @see Materialized + */ +public final class MaterializedX extends + ModifierChain, Configurator, MaterializedX> { + + private MaterializedX(final Function> initializer) { + super(initializer); + } + + /** + * Create an instance of {@code MaterializedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code MaterializedX} + * @param type of keys + * @param type of values + * @param type of state store + */ + public static MaterializedX keySerde( + final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code MaterializedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code MaterializedX} + * @param type of keys + * @param type of values + * @param type of state store + */ + public static MaterializedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * Create an instance of {@code MaterializedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code MaterializedX} + * @param type of keys + * @param type of values + * @param type of state store + */ + public static MaterializedX valueSerde( + final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde); + } + + /** + * Create an instance of {@code MaterializedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code MaterializedX} + * @param type of keys + * @param type of values + * @param type of state store + */ + public static MaterializedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Materialized#with(Serde, Serde) + */ + public static MaterializedX with( + final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new MaterializedX<>(configurator -> Materialized.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde))); + } + + /** + * @see Materialized#with(Serde, Serde) + */ + public static MaterializedX with( + final Serde keySerde, + final Serde valueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + + /** + * @see Materialized#as(String) + */ + public static MaterializedX as(final String storeName) { + return new MaterializedX<>(configurator -> Materialized.as(storeName)); + } + + /** + * @see Materialized#as(DslStoreSuppliers) + */ + public static MaterializedX as( + final DslStoreSuppliers storeSuppliers) { + return new MaterializedX<>(configurator -> Materialized.as(storeSuppliers)); + } + + /** + * @see Materialized#as(WindowBytesStoreSupplier) + */ + public static MaterializedX> as(final WindowBytesStoreSupplier supplier) { + return new MaterializedX<>(configurator -> Materialized.as(supplier)); + } + + /** + * @see Materialized#as(SessionBytesStoreSupplier) + */ + public static MaterializedX> as(final SessionBytesStoreSupplier supplier) { + return new MaterializedX<>(configurator -> Materialized.as(supplier)); + } + + /** + * @see Materialized#as(KeyValueBytesStoreSupplier) + */ + public static MaterializedX> as( + final KeyValueBytesStoreSupplier supplier) { + return new MaterializedX<>(configurator -> Materialized.as(supplier)); + } + + /** + * @see Materialized#withKeySerde(Serde) + */ + public MaterializedX withKeySerde(final Preconfigured> keySerde) { + return this.modify( + (materialized, configurator) -> materialized.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Materialized#withKeySerde(Serde) + */ + public MaterializedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Materialized#withValueSerde(Serde) + */ + public MaterializedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify((materialized, configurator) -> materialized.withValueSerde( + configurator.configureForValues(valueSerde))); + } + + /** + * @see Materialized#withValueSerde(Serde) + */ + public MaterializedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Materialized#withRetention(Duration) + */ + public MaterializedX withRetention(final Duration retention) { + return this.modify(materialized -> materialized.withRetention(retention)); + } + + /** + * @see Materialized#withStoreType(DslStoreSuppliers) + */ + public MaterializedX withStoreType(final DslStoreSuppliers storeSuppliers) { + return this.modify(materialized -> materialized.withStoreType(storeSuppliers)); + } + + /** + * @see Materialized#withLoggingEnabled(Map) + */ + public MaterializedX withLoggingEnabled(final Map config) { + return this.modify(materialized -> materialized.withLoggingEnabled(config)); + } + + /** + * @see Materialized#withLoggingDisabled() + */ + public MaterializedX withLoggingDisabled() { + return this.modify(Materialized::withLoggingDisabled); + } + + /** + * @see Materialized#withCachingDisabled() + */ + public MaterializedX withCachingDisabled() { + return this.modify(Materialized::withCachingDisabled); + } + + /** + * @see Materialized#withCachingEnabled() + */ + public MaterializedX withCachingEnabled() { + return this.modify(Materialized::withCachingEnabled); + } + + @Override + protected MaterializedX newInstance(final Function> initializer) { + return new MaterializedX<>(initializer); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ModifierChain.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ModifierChain.java new file mode 100644 index 00000000..ffac90b8 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ModifierChain.java @@ -0,0 +1,62 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.BiFunction; +import java.util.function.Function; +import lombok.AccessLevel; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor(access = AccessLevel.PROTECTED) +abstract class ModifierChain { + + private final @NonNull Function initializer; + + protected abstract SELF newInstance(Function initializer); + + protected final SELF modify(final BiFunction modifier) { + return this.newInstance(context -> this.modify(context, modifier)); + } + + protected final SELF modify(final Function modifier) { + return this.newInstance(context -> this.modify(context, modifier)); + } + + final T configure(final C context) { + return this.initializer.apply(context); + } + + private T modify(final C context, final BiFunction modifier) { + final T configure = this.configure(context); + return modifier.apply(configure, context); + } + + private T modify(final C context, final Function modifier) { + final T configure = this.configure(context); + return modifier.apply(configure); + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducedX.java new file mode 100644 index 00000000..f53263f9 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/ProducedX.java @@ -0,0 +1,149 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.processor.StreamPartitioner; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Produced} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Produced + */ +public final class ProducedX extends ModifierChain, Configurator, ProducedX> { + + private ProducedX(final Function> initializer) { + super(initializer); + } + + /** + * @see Produced#keySerde(Serde) + */ + public static ProducedX keySerde(final Preconfigured> keySerde) { + return new ProducedX<>(configurator -> Produced.keySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Produced#keySerde(Serde) + */ + public static ProducedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Produced#valueSerde(Serde) + */ + public static ProducedX valueSerde(final Preconfigured> valueSerde) { + return new ProducedX<>(configurator -> Produced.valueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Produced#valueSerde(Serde) + */ + public static ProducedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Produced#with(Serde, Serde) + */ + public static ProducedX with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new ProducedX<>(configurator -> Produced.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde))); + } + + /** + * @see Produced#with(Serde, Serde) + */ + public static ProducedX with(final Serde keySerde, final Serde valueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + + /** + * @see Produced#as(String) + */ + public static ProducedX as(final String processorName) { + return new ProducedX<>(configurator -> Produced.as(processorName)); + } + + /** + * @see Produced#streamPartitioner(StreamPartitioner) + */ + public static ProducedX streamPartitioner(final StreamPartitioner partitioner) { + return new ProducedX<>(configurator -> Produced.streamPartitioner(partitioner)); + } + + /** + * @see Produced#withKeySerde(Serde) + */ + public ProducedX withKeySerde(final Preconfigured> keySerde) { + return this.modify((produced, configurator) -> produced.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Produced#withKeySerde(Serde) + */ + public ProducedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Produced#withValueSerde(Serde) + */ + public ProducedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify( + (produced, configurator) -> produced.withValueSerde(configurator.configureForValues(valueSerde))); + } + + /** + * @see Produced#withValueSerde(Serde) + */ + public ProducedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Produced#withStreamPartitioner(StreamPartitioner) + */ + public ProducedX withStreamPartitioner(final StreamPartitioner partitioner) { + return this.modify(produced -> produced.withStreamPartitioner(partitioner)); + } + + /** + * @see Produced#withName(String) + */ + public ProducedX withName(final String name) { + return this.modify(produced -> produced.withName(name)); + } + + @Override + protected ProducedX newInstance(final Function> initializer) { + return new ProducedX<>(initializer); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/RepartitionedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/RepartitionedX.java new file mode 100644 index 00000000..615a6fb0 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/RepartitionedX.java @@ -0,0 +1,180 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.Repartitioned; +import org.apache.kafka.streams.processor.StreamPartitioner; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Repartitioned} using {@link Configurator} + * @param type of keys + * @param type of values + * @see Repartitioned + */ +public final class RepartitionedX extends ModifierChain, Configurator, RepartitionedX> { + + private RepartitionedX(final Function> initializer) { + super(initializer); + } + + /** + * Create an instance of {@code RepartitionedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code RepartitionedX} + * @param type of keys + * @param type of values + */ + public static RepartitionedX keySerde(final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code RepartitionedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code RepartitionedX} + * @param type of keys + * @param type of values + */ + public static RepartitionedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * Create an instance of {@code RepartitionedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code RepartitionedX} + * @param type of keys + * @param type of values + */ + public static RepartitionedX valueSerde(final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde); + } + + /** + * Create an instance of {@code RepartitionedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code RepartitionedX} + * @param type of keys + * @param type of values + */ + public static RepartitionedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Repartitioned#with(Serde, Serde) + */ + public static RepartitionedX with(final Preconfigured> keySerde, + final Preconfigured> valueSerde) { + return new RepartitionedX<>(configurator -> Repartitioned.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde))); + } + + /** + * @see Repartitioned#with(Serde, Serde) + */ + public static RepartitionedX with(final Serde keySerde, final Serde valueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + + /** + * @see Repartitioned#as(String) + */ + public static RepartitionedX as(final String name) { + return new RepartitionedX<>(configurator -> Repartitioned.as(name)); + } + + /** + * @see Repartitioned#numberOfPartitions(int) + */ + public static RepartitionedX numberOfPartitions(final int numberOfPartitions) { + return new RepartitionedX<>(configurator -> Repartitioned.numberOfPartitions(numberOfPartitions)); + } + + /** + * @see Repartitioned#streamPartitioner(StreamPartitioner) + */ + public static RepartitionedX streamPartitioner(final StreamPartitioner partitioner) { + return new RepartitionedX<>(configurator -> Repartitioned.streamPartitioner(partitioner)); + } + + /** + * @see Repartitioned#withNumberOfPartitions(int) + */ + public RepartitionedX withNumberOfPartitions(final int numberOfPartitions) { + return this.modify(repartitioned -> repartitioned.withNumberOfPartitions(numberOfPartitions)); + } + + /** + * @see Repartitioned#withKeySerde(Serde) + */ + public RepartitionedX withKeySerde(final Preconfigured> keySerde) { + return this.modify( + (repartitioned, configurator) -> repartitioned.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see Repartitioned#withKeySerde(Serde) + */ + public RepartitionedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see Repartitioned#withValueSerde(Serde) + */ + public RepartitionedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify((repartitioned, configurator) -> repartitioned.withValueSerde( + configurator.configureForValues(valueSerde))); + } + + /** + * @see Repartitioned#withValueSerde(Serde) + */ + public RepartitionedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see Repartitioned#withStreamPartitioner(StreamPartitioner) + */ + public RepartitionedX withStreamPartitioner(final StreamPartitioner partitioner) { + return this.modify(repartitioned -> repartitioned.withStreamPartitioner(partitioner)); + } + + /** + * @see Repartitioned#withName(String) + */ + public RepartitionedX withName(final String name) { + return this.modify(repartitioned -> repartitioned.withName(name)); + } + + @Override + protected RepartitionedX newInstance(final Function> initializer) { + return new RepartitionedX<>(initializer); + } +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java index 38e6bad0..7ac62fb2 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamX.java @@ -56,7 +56,7 @@ KTableX, VOut> aggregate(Initializer initializer, Merger, VOut> aggregate(Initializer initializer, Merger sessionMerger, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, VOut> aggregate(Initializer initializer, Merger sessionMerger, @@ -66,5 +66,5 @@ KTableX, VOut> aggregate(Initializer initializer, Merger, VOut> aggregate(Initializer initializer, Merger sessionMerger, - Named named, AutoMaterialized> materialized); + Named named, MaterializedX> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java index 9bd8baff..2c942380 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedCogroupedStreamXImpl.java @@ -64,7 +64,7 @@ public KTableX, V> aggregate(final Initializer initializer, @Override public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, sessionMerger, materialized.configure(this.context.getConfigurator())); } @@ -78,7 +78,7 @@ public KTableX, V> aggregate(final Initializer initializer, @Override public KTableX, V> aggregate(final Initializer initializer, final Merger sessionMerger, - final Named named, final AutoMaterialized> materialized) { + final Named named, final MaterializedX> materialized) { return this.aggregate(initializer, sessionMerger, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java index d004c338..8d29554f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedKStreamX.java @@ -56,7 +56,7 @@ public interface SessionWindowedKStreamX extends SessionWindowedKStream, Long> count(AutoMaterialized> materialized); + KTableX, Long> count(MaterializedX> materialized); @Override KTableX, Long> count(Named named, @@ -66,7 +66,7 @@ KTableX, Long> count(Named named, * @see #count(Named, Materialized) */ KTableX, Long> count(Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, VR> aggregate(Initializer initializer, @@ -89,7 +89,7 @@ KTableX, VR> aggregate(Initializer initializer, KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, VR> aggregate(Initializer initializer, @@ -103,7 +103,7 @@ KTableX, VR> aggregate(Initializer initializer, KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, Merger sessionMerger, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, V> reduce(Reducer reducer); @@ -119,7 +119,7 @@ KTableX, V> reduce(Reducer reducer, * @see #reduce(Reducer, Materialized) */ KTableX, V> reduce(Reducer reducer, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, V> reduce(Reducer reducer, Named named, @@ -129,7 +129,7 @@ KTableX, V> reduce(Reducer reducer, Named named, * @see #reduce(Reducer, Named, Materialized) */ KTableX, V> reduce(Reducer reducer, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override SessionWindowedKStreamX emitStrategy(EmitStrategy emitStrategy); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java index 1b21a728..e6cfe920 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/SessionWindowedStreamXImpl.java @@ -62,7 +62,7 @@ public KTableX, Long> count( @Override public KTableX, Long> count( - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -74,7 +74,7 @@ public KTableX, Long> count(final Named named, @Override public KTableX, Long> count(final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -101,7 +101,7 @@ public KTableX, VR> aggregate(final Initializer initializer @Override public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, aggregator, sessionMerger, materialized.configure(this.context.getConfigurator())); } @@ -118,7 +118,7 @@ public KTableX, VR> aggregate(final Initializer initializer public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Merger sessionMerger, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, aggregator, sessionMerger, named, materialized.configure(this.context.getConfigurator())); } @@ -141,7 +141,7 @@ public KTableX, V> reduce(final Reducer reducer, @Override public KTableX, V> reduce(final Reducer reducer, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @@ -153,7 +153,7 @@ public KTableX, V> reduce(final Reducer reducer, final Named name @Override public KTableX, V> reduce(final Reducer reducer, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStores.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StoresX.java similarity index 67% rename from streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStores.java rename to streams-bootstrap-core/src/main/java/com/bakdata/kafka/StoresX.java index 02a94d5e..0c30f551 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/AutoStores.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StoresX.java @@ -44,7 +44,7 @@ * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link Stores} using {@link Configurator} */ @RequiredArgsConstructor -public class AutoStores { +public class StoresX { private final @NonNull Configurator configurator; @@ -57,6 +57,14 @@ public StoreBuilder> sessionStoreBuilder(final Session this.configurator.configureForValues(valueSerde)); } + /** + * @see Stores#sessionStoreBuilder(SessionBytesStoreSupplier, Serde, Serde) + */ + public StoreBuilder> sessionStoreBuilder(final SessionBytesStoreSupplier supplier, + final Serde keySerde, final Serde valueSerde) { + return this.sessionStoreBuilder(supplier, Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + /** * @see Stores#timestampedWindowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) */ @@ -67,6 +75,15 @@ public StoreBuilder> timestampedWindowStoreB this.configurator.configureForValues(valueSerde)); } + /** + * @see Stores#timestampedWindowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) + */ + public StoreBuilder> timestampedWindowStoreBuilder( + final WindowBytesStoreSupplier supplier, final Serde keySerde, final Serde valueSerde) { + return this.timestampedWindowStoreBuilder(supplier, Preconfigured.create(keySerde), + Preconfigured.create(valueSerde)); + } + /** * @see Stores#windowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) */ @@ -76,6 +93,14 @@ public StoreBuilder> windowStoreBuilder(final WindowByt this.configurator.configureForValues(valueSerde)); } + /** + * @see Stores#windowStoreBuilder(WindowBytesStoreSupplier, Serde, Serde) + */ + public StoreBuilder> windowStoreBuilder(final WindowBytesStoreSupplier supplier, + final Serde keySerde, final Serde valueSerde) { + return this.windowStoreBuilder(supplier, Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } + /** * @see Stores#versionedKeyValueStoreBuilder(VersionedBytesStoreSupplier, Serde, Serde) */ @@ -86,6 +111,15 @@ public StoreBuilder> versionedKeyValueStoreB this.configurator.configureForValues(valueSerde)); } + /** + * @see Stores#versionedKeyValueStoreBuilder(VersionedBytesStoreSupplier, Serde, Serde) + */ + public StoreBuilder> versionedKeyValueStoreBuilder( + final VersionedBytesStoreSupplier supplier, final Serde keySerde, final Serde valueSerde) { + return this.versionedKeyValueStoreBuilder(supplier, Preconfigured.create(keySerde), + Preconfigured.create(valueSerde)); + } + /** * @see Stores#timestampedKeyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) */ @@ -96,6 +130,16 @@ public StoreBuilder> timestampedKeyValueSt this.configurator.configureForValues(valueSerde)); } + /** + * @see Stores#timestampedKeyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) + */ + public StoreBuilder> timestampedKeyValueStoreBuilder( + final KeyValueBytesStoreSupplier supplier, final Serde keySerde, + final Serde valueSerde) { + return this.timestampedKeyValueStoreBuilder(supplier, Preconfigured.create(keySerde), + Preconfigured.create(valueSerde)); + } + /** * @see Stores#keyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) */ @@ -104,4 +148,12 @@ public StoreBuilder> keyValueStoreBuilder(final KeyVa return Stores.keyValueStoreBuilder(supplier, this.configurator.configureForKeys(keySerde), this.configurator.configureForValues(valueSerde)); } + + /** + * @see Stores#keyValueStoreBuilder(KeyValueBytesStoreSupplier, Serde, Serde) + */ + public StoreBuilder> keyValueStoreBuilder(final KeyValueBytesStoreSupplier supplier, + final Serde keySerde, final Serde valueSerde) { + return this.keyValueStoreBuilder(supplier, Preconfigured.create(keySerde), Preconfigured.create(valueSerde)); + } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamJoinedX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamJoinedX.java new file mode 100644 index 00000000..c6ba4423 --- /dev/null +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/StreamJoinedX.java @@ -0,0 +1,266 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import java.util.Map; +import java.util.function.Function; +import org.apache.kafka.common.serialization.Serde; +import org.apache.kafka.streams.kstream.StreamJoined; +import org.apache.kafka.streams.state.DslStoreSuppliers; +import org.apache.kafka.streams.state.WindowBytesStoreSupplier; + +/** + * Use {@link Preconfigured} to lazily configure {@link Serde} for {@link StreamJoined} using {@link Configurator} + * @param type of keys + * @param this value type + * @param other value type + * @see StreamJoined + */ +public final class StreamJoinedX extends + ModifierChain, Configurator, StreamJoinedX> { + + private StreamJoinedX(final Function> initializer) { + super(initializer); + } + + /** + * Create an instance of {@code StreamJoinedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code StreamJoinedX} + * @param type of keys + * @param this value type + * @param other value type + */ + public static StreamJoinedX keySerde( + final Preconfigured> keySerde) { + return with(keySerde, Preconfigured.defaultSerde(), Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code StreamJoinedX} with provided key serde + * @param keySerde Serde to use for keys + * @return a new instance of {@code StreamJoinedX} + * @param type of keys + * @param this value type + * @param other value type + */ + public static StreamJoinedX keySerde(final Serde keySerde) { + return keySerde(Preconfigured.create(keySerde)); + } + + /** + * Create an instance of {@code StreamJoinedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code StreamJoinedX} + * @param type of keys + * @param this value type + * @param other value type + */ + public static StreamJoinedX valueSerde( + final Preconfigured> valueSerde) { + return with(Preconfigured.defaultSerde(), valueSerde, Preconfigured.defaultSerde()); + } + + /** + * Create an instance of {@code StreamJoinedX} with provided value serde + * @param valueSerde Serde to use for values + * @return a new instance of {@code StreamJoinedX} + * @param type of keys + * @param this value type + * @param other value type + */ + public static StreamJoinedX valueSerde(final Serde valueSerde) { + return valueSerde(Preconfigured.create(valueSerde)); + } + + /** + * Create an instance of {@code StreamJoinedX} with provided other value serde + * @param otherValueSerde Serde to use for other values + * @return a new instance of {@code StreamJoinedX} + * @param type of keys + * @param this value type + * @param other value type + */ + public static StreamJoinedX otherValueSerde( + final Preconfigured> otherValueSerde) { + return with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde(), otherValueSerde); + } + + /** + * Create an instance of {@code StreamJoinedX} with provided other value serde + * @param otherValueSerde Serde to use for other values + * @return a new instance of {@code StreamJoinedX} + * @param type of keys + * @param this value type + * @param other value type + */ + public static StreamJoinedX otherValueSerde(final Serde otherValueSerde) { + return otherValueSerde(Preconfigured.create(otherValueSerde)); + } + + /** + * @see StreamJoined#with(Serde, Serde, Serde) + */ + public static StreamJoinedX with( + final Preconfigured> keySerde, + final Preconfigured> valueSerde, + final Preconfigured> otherValueSerde) { + return new StreamJoinedX<>(configurator -> StreamJoined.with(configurator.configureForKeys(keySerde), + configurator.configureForValues(valueSerde), configurator.configureForValues(otherValueSerde))); + } + + /** + * @see StreamJoined#with(Serde, Serde, Serde) + */ + public static StreamJoinedX with( + final Serde keySerde, + final Serde valueSerde, + final Serde otherValueSerde) { + return with(Preconfigured.create(keySerde), Preconfigured.create(valueSerde), + Preconfigured.create(otherValueSerde)); + } + + /** + * @see StreamJoined#as(String) + */ + public static StreamJoinedX as(final String storeName) { + return new StreamJoinedX<>(configurator -> StreamJoined.as(storeName)); + } + + /** + * @see StreamJoined#with(DslStoreSuppliers) + */ + public static StreamJoinedX with(final DslStoreSuppliers storeSuppliers) { + return new StreamJoinedX<>(configurator -> StreamJoined.with(storeSuppliers)); + } + + /** + * @see StreamJoined#with(WindowBytesStoreSupplier, WindowBytesStoreSupplier) + */ + public static StreamJoinedX with(final WindowBytesStoreSupplier storeSupplier, + final WindowBytesStoreSupplier otherStoreSupplier) { + return new StreamJoinedX<>(configurator -> StreamJoined.with(storeSupplier, otherStoreSupplier)); + } + + /** + * @see StreamJoined#withKeySerde(Serde) + */ + public StreamJoinedX withKeySerde(final Preconfigured> keySerde) { + return this.modify( + (streamJoined, configurator) -> streamJoined.withKeySerde(configurator.configureForKeys(keySerde))); + } + + /** + * @see StreamJoined#withKeySerde(Serde) + */ + public StreamJoinedX withKeySerde(final Serde keySerde) { + return this.withKeySerde(Preconfigured.create(keySerde)); + } + + /** + * @see StreamJoined#withValueSerde(Serde) + */ + public StreamJoinedX withValueSerde(final Preconfigured> valueSerde) { + return this.modify((streamJoined, configurator) -> streamJoined.withValueSerde( + configurator.configureForValues(valueSerde))); + } + + /** + * @see StreamJoined#withValueSerde(Serde) + */ + public StreamJoinedX withValueSerde(final Serde valueSerde) { + return this.withValueSerde(Preconfigured.create(valueSerde)); + } + + /** + * @see StreamJoined#withOtherValueSerde(Serde) + */ + public StreamJoinedX withOtherValueSerde(final Preconfigured> otherValueSerde) { + return this.modify((streamJoined, configurator) -> streamJoined.withOtherValueSerde( + configurator.configureForValues(otherValueSerde))); + } + + /** + * @see StreamJoined#withOtherValueSerde(Serde) + */ + public StreamJoinedX withOtherValueSerde(final Serde otherValueSerde) { + return this.withOtherValueSerde(Preconfigured.create(otherValueSerde)); + } + + /** + * @see StreamJoined#withDslStoreSuppliers(DslStoreSuppliers) + */ + public StreamJoinedX withDslStoreSuppliers(final DslStoreSuppliers dslStoreSuppliers) { + return this.modify(streamJoined -> streamJoined.withDslStoreSuppliers(dslStoreSuppliers)); + } + + /** + * @see StreamJoined#withName(String) + */ + public StreamJoinedX withName(final String name) { + return this.modify(streamJoined -> streamJoined.withName(name)); + } + + /** + * @see StreamJoined#withStoreName(String) + */ + public StreamJoinedX withStoreName(final String storeName) { + return this.modify(streamJoined -> streamJoined.withStoreName(storeName)); + } + + /** + * @see StreamJoined#withLoggingEnabled(Map) + */ + public StreamJoinedX withLoggingEnabled(final Map config) { + return this.modify(streamJoined -> streamJoined.withLoggingEnabled(config)); + } + + /** + * @see StreamJoined#withLoggingDisabled() + */ + public StreamJoinedX withLoggingDisabled() { + return this.modify(StreamJoined::withLoggingDisabled); + } + + /** + * @see StreamJoined#withThisStoreSupplier(WindowBytesStoreSupplier) + */ + public StreamJoinedX withThisStoreSupplier(final WindowBytesStoreSupplier thisStoreSupplier) { + return this.modify(streamJoined -> streamJoined.withThisStoreSupplier(thisStoreSupplier)); + } + + /** + * @see StreamJoined#withOtherStoreSupplier(WindowBytesStoreSupplier) + */ + public StreamJoinedX withOtherStoreSupplier(final WindowBytesStoreSupplier otherStoreSupplier) { + return this.modify(streamJoined -> streamJoined.withOtherStoreSupplier(otherStoreSupplier)); + } + + @Override + protected StreamJoinedX newInstance(final Function> initializer) { + return new StreamJoinedX<>(initializer); + } + +} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java index 8a60012c..42bcdf34 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamX.java @@ -54,7 +54,7 @@ KTableX, VOut> aggregate(Initializer initializer, * @see #aggregate(Initializer, Materialized) */ KTableX, VOut> aggregate(Initializer initializer, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, VOut> aggregate(Initializer initializer, Named named, @@ -64,5 +64,5 @@ KTableX, VOut> aggregate(Initializer initializer, Named named, * @see #aggregate(Initializer, Named, Materialized) */ KTableX, VOut> aggregate(Initializer initializer, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java index f4d96319..9c7de04a 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedCogroupedStreamXImpl.java @@ -58,7 +58,7 @@ public KTableX, V> aggregate(final Initializer initializer, @Override public KTableX, V> aggregate(final Initializer initializer, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, materialized.configure(this.context.getConfigurator())); } @@ -70,7 +70,7 @@ public KTableX, V> aggregate(final Initializer initializer, final @Override public KTableX, V> aggregate(final Initializer initializer, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, named, materialized.configure(this.context.getConfigurator())); } } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java index f275bf52..e6f5ee73 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedKStreamX.java @@ -55,7 +55,7 @@ public interface TimeWindowedKStreamX extends TimeWindowedKStream { /** * @see #count(Materialized) */ - KTableX, Long> count(AutoMaterialized> materialized); + KTableX, Long> count(MaterializedX> materialized); @Override KTableX, Long> count(Named named, @@ -65,7 +65,7 @@ KTableX, Long> count(Named named, * @see #count(Named, Materialized) */ KTableX, Long> count(Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, VR> aggregate(Initializer initializer, @@ -86,7 +86,7 @@ KTableX, VR> aggregate(Initializer initializer, */ KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, VR> aggregate(Initializer initializer, @@ -98,7 +98,7 @@ KTableX, VR> aggregate(Initializer initializer, */ KTableX, VR> aggregate(Initializer initializer, Aggregator aggregator, - Named named, AutoMaterialized> materialized); + Named named, MaterializedX> materialized); @Override KTableX, V> reduce(Reducer reducer); @@ -114,7 +114,7 @@ KTableX, V> reduce(Reducer reducer, * @see #reduce(Reducer, Materialized) */ KTableX, V> reduce(Reducer reducer, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override KTableX, V> reduce(Reducer reducer, Named named, @@ -124,7 +124,7 @@ KTableX, V> reduce(Reducer reducer, Named named, * @see #reduce(Reducer, Named, Materialized) */ KTableX, V> reduce(Reducer reducer, Named named, - AutoMaterialized> materialized); + MaterializedX> materialized); @Override TimeWindowedKStreamX emitStrategy(EmitStrategy emitStrategy); diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java index b7aba96b..37053699 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TimeWindowedStreamXImpl.java @@ -61,7 +61,7 @@ public KTableX, Long> count( @Override public KTableX, Long> count( - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(materialized.configure(this.context.getConfigurator())); } @@ -73,7 +73,7 @@ public KTableX, Long> count(final Named named, @Override public KTableX, Long> count(final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.count(named, materialized.configure(this.context.getConfigurator())); } @@ -99,7 +99,7 @@ public KTableX, VR> aggregate(final Initializer initializer @Override public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, aggregator, materialized.configure(this.context.getConfigurator())); } @@ -113,7 +113,7 @@ public KTableX, VR> aggregate(final Initializer initializer @Override public KTableX, VR> aggregate(final Initializer initializer, final Aggregator aggregator, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.aggregate(initializer, aggregator, named, materialized.configure(this.context.getConfigurator())); } @@ -135,7 +135,7 @@ public KTableX, V> reduce(final Reducer reducer, @Override public KTableX, V> reduce(final Reducer reducer, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(reducer, materialized.configure(this.context.getConfigurator())); } @@ -147,7 +147,7 @@ public KTableX, V> reduce(final Reducer reducer, final Named name @Override public KTableX, V> reduce(final Reducer reducer, final Named named, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.reduce(reducer, named, materialized.configure(this.context.getConfigurator())); } diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java index 05667802..64464b3b 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/TopologyBuilder.java @@ -72,7 +72,7 @@ public KStreamX stream(final String topic, final Consumed con /** * @see StreamsBuilder#stream(String, Consumed) */ - public KStreamX stream(final String topic, final AutoConsumed consumed) { + public KStreamX stream(final String topic, final ConsumedX consumed) { return this.stream(topic, consumed.configure(this.createConfigurator())); } @@ -94,7 +94,7 @@ public KStreamX stream(final Collection topics, final Consu * @see StreamsBuilder#stream(Collection, Consumed) */ public KStreamX stream(final Collection topics, - final AutoConsumed consumed) { + final ConsumedX consumed) { return this.stream(topics, consumed.configure(this.createConfigurator())); } @@ -115,7 +115,7 @@ public KStreamX stream(final Pattern topicPattern, final Consumed KStreamX stream(final Pattern topicPattern, final AutoConsumed consumed) { + public KStreamX stream(final Pattern topicPattern, final ConsumedX consumed) { return this.stream(topicPattern, consumed.configure(this.createConfigurator())); } @@ -139,7 +139,7 @@ public KStreamX streamInput(final Consumed consumed) { * @param type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public KStreamX streamInput(final AutoConsumed consumed) { + public KStreamX streamInput(final ConsumedX consumed) { return this.streamInput(consumed.configure(this.createConfigurator())); } @@ -176,7 +176,7 @@ public KStreamX streamInput(final String label, final Consumed type of values * @see StreamsBuilder#stream(Collection, Consumed) */ - public KStreamX streamInput(final String label, final AutoConsumed consumed) { + public KStreamX streamInput(final String label, final ConsumedX consumed) { return this.streamInput(label, consumed.configure(this.createConfigurator())); } @@ -212,7 +212,7 @@ public KStreamX streamInputPattern(final Consumed consumed) { * @param type of values * @see StreamsBuilder#stream(Pattern, Consumed) */ - public KStreamX streamInputPattern(final AutoConsumed consumed) { + public KStreamX streamInputPattern(final ConsumedX consumed) { return this.streamInputPattern(consumed.configure(this.createConfigurator())); } @@ -250,7 +250,7 @@ public KStreamX streamInputPattern(final String label, final Consum * @see StreamsBuilder#stream(Pattern, Consumed) */ public KStreamX streamInputPattern(final String label, - final AutoConsumed consumed) { + final ConsumedX consumed) { return this.streamInputPattern(label, consumed.configure(this.createConfigurator())); } @@ -283,7 +283,7 @@ public KTableX table(final String topic, final Consumed consu /** * @see StreamsBuilder#table(String, Consumed) */ - public KTableX table(final String topic, final AutoConsumed consumed) { + public KTableX table(final String topic, final ConsumedX consumed) { return this.table(topic, consumed.configure(this.createConfigurator())); } @@ -299,7 +299,7 @@ public KTableX table(final String topic, * @see StreamsBuilder#table(String, Materialized) */ public KTableX table(final String topic, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.table(topic, materialized.configure(this.createConfigurator())); } @@ -314,10 +314,10 @@ public KTableX table(final String topic, final Consumed consu /** * @see StreamsBuilder#table(String, Consumed, Materialized) */ - public KTableX table(final String topic, final AutoConsumed consumed, - final AutoMaterialized> materialized) { + public KTableX table(final String topic, final ConsumedX consumed, + final Materialized> materialized) { final Configurator configurator = this.createConfigurator(); - return this.table(topic, consumed.configure(configurator), materialized.configure(configurator)); + return this.table(topic, consumed.configure(configurator), materialized); } /** @@ -337,7 +337,7 @@ public GlobalKTable globalTable(final String topic, final Consumed< /** * @see StreamsBuilder#globalTable(String, Consumed) */ - public GlobalKTable globalTable(final String topic, final AutoConsumed consumed) { + public GlobalKTable globalTable(final String topic, final ConsumedX consumed) { return this.globalTable(topic, consumed.configure(this.createConfigurator())); } @@ -353,7 +353,7 @@ public GlobalKTable globalTable(final String topic, * @see StreamsBuilder#globalTable(String, Materialized) */ public GlobalKTable globalTable(final String topic, - final AutoMaterialized> materialized) { + final MaterializedX> materialized) { return this.globalTable(topic, materialized.configure(this.createConfigurator())); } @@ -368,10 +368,10 @@ public GlobalKTable globalTable(final String topic, final Consumed< /** * @see StreamsBuilder#globalTable(String, Consumed, Materialized) */ - public GlobalKTable globalTable(final String topic, final AutoConsumed consumed, - final AutoMaterialized> materialized) { + public GlobalKTable globalTable(final String topic, final ConsumedX consumed, + final Materialized> materialized) { final Configurator configurator = this.createConfigurator(); - return this.globalTable(topic, consumed.configure(configurator), materialized.configure(configurator)); + return this.globalTable(topic, consumed.configure(configurator), materialized); } /** @@ -395,7 +395,7 @@ public TopologyBuilder addGlobalStore(final StoreBuilder storeBuil * @see StreamsBuilder#addGlobalStore(StoreBuilder, String, Consumed, ProcessorSupplier) */ public TopologyBuilder addGlobalStore(final StoreBuilder storeBuilder, final String topic, - final AutoConsumed consumed, final ProcessorSupplier stateUpdateSupplier) { + final ConsumedX consumed, final ProcessorSupplier stateUpdateSupplier) { return this.addGlobalStore(storeBuilder, topic, consumed.configure(this.createConfigurator()), stateUpdateSupplier); } @@ -427,10 +427,10 @@ public StreamsContext getContext() { /** * Create stores using application context to lazily configures Serdes - * @return {@link AutoStores} + * @return {@link StoresX} */ - public AutoStores stores() { - return new AutoStores(this.createConfigurator()); + public StoresX stores() { + return new StoresX(this.createConfigurator()); } Topology build() { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java index 4dd39e75..b1c6aecf 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedKStreamXTest.java @@ -24,8 +24,6 @@ package com.bakdata.kafka; -import static com.bakdata.kafka.KStreamXTest.startApp; - import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.util.Map; import java.util.function.Function; @@ -37,7 +35,7 @@ class BranchedKStreamXTest { @Test void shouldBranch() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -47,21 +45,20 @@ public void buildTopology(final TopologyBuilder builder) { branches.values().iterator().next().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldBranchBranched() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -69,21 +66,20 @@ public void buildTopology(final TopologyBuilder builder) { .branch((k, v) -> "foo".equals(k), Branched.withConsumer(branch -> branch.to("output"))); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldBranchBranchedX() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -92,29 +88,28 @@ public void buildTopology(final TopologyBuilder builder) { .defaultBranchX(Branched.withConsumer(s -> s.to("default_output"))); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input() - .add("foo", "bar") - .add("baz", "qux"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.streamOutput("default_output") - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar") + .add("baz", "qux"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.streamOutput("default_output") + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldUseDefaultBranch() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -123,21 +118,20 @@ public void buildTopology(final TopologyBuilder builder) { branches.values().iterator().next().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldUseDefaultBranchX() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -146,23 +140,22 @@ public void buildTopology(final TopologyBuilder builder) { branches.values().iterator().next().toOutputTopic(); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldUseDefaultBranchBranched() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -170,21 +163,20 @@ public void buildTopology(final TopologyBuilder builder) { .defaultBranch(Branched.withConsumer(branch -> branch.to("output"))); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldUseDefaultBranchXBranched() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -193,23 +185,22 @@ public void buildTopology(final TopologyBuilder builder) { branches.values().iterator().next().toOutputTopic(); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldUseDefaultBranchBranchedX() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -218,23 +209,22 @@ public void buildTopology(final TopologyBuilder builder) { branches.values().iterator().next().toOutputTopic(); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldUseNoDefaultBranchX() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -244,17 +234,16 @@ public void buildTopology(final TopologyBuilder builder) { branches.values().iterator().next().toOutputTopic(); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index 15aaa022..4b99d0de 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -24,8 +24,6 @@ package com.bakdata.kafka; -import static com.bakdata.kafka.KStreamXTest.startApp; - import com.bakdata.fluent_kafka_streams_tests.TestTopology; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.kstream.Named; @@ -35,7 +33,7 @@ class KGroupedStreamXTest { @Test void shouldCount() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -44,26 +42,25 @@ public void buildTopology(final TopologyBuilder builder) { counted.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey("foo") - .hasValue(1L) - .expectNextRecord() - .hasKey("foo") - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldCountNamed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -72,102 +69,91 @@ public void buildTopology(final TopologyBuilder builder) { counted.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey("foo") - .hasValue(1L) - .expectNextRecord() - .hasKey("foo") - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldCountUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX counted = - grouped.count(AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - counted.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + grouped.count(MaterializedX.with(Serdes.Long(), Serdes.Long())); + counted.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(1L) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldCountNamedUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KTableX counted = - grouped.count(Named.as("count"), AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - counted.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); + final KTableX counted = grouped.count(Named.as("count"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + counted.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(1L) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldReduce() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -176,101 +162,90 @@ public void buildTopology(final TopologyBuilder builder) { reduced.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldReduceUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX reduced = grouped.reduce(Long::sum, - AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - reduced.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + MaterializedX.with(Serdes.Long(), Serdes.Long())); + reduced.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldReduceNamedUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX reduced = grouped.reduce(Long::sum, Named.as("reduce"), - AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - reduced.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + MaterializedX.with(Serdes.Long(), Serdes.Long())); + reduced.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldAggregate() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -280,97 +255,88 @@ public void buildTopology(final TopologyBuilder builder) { aggregated.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldAggregateUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX aggregated = grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, - AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - aggregated.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + MaterializedX.with(Serdes.Long(), Serdes.Long())); + aggregated.toStream() + .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldAggregateNamedUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX aggregated = - grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, - Named.as("aggregate"), AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - aggregated.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, Named.as("aggregate"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + aggregated.toStream() + .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index f7fe0f66..48da3bb6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -73,433 +73,400 @@ class KStreamXTest { @InjectSoftAssertions private SoftAssertions softly; - static TestTopology startApp(final StreamsApp app, final StreamsTopicConfig topicConfig) { - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig)); - final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); - topology.start(); - return topology; - } - @Test void shouldWriteToOutput() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.toOutputTopic(); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldWriteToOutputUsingProduced() { - final StreamsApp app = new SimpleApp() { - @Override - public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.toOutputTopic(AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .outputTopic("output") - .build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.toOutputTopic(ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldWriteToLabeledOutput() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.toOutputTopic("label"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .labeledOutputTopics(Map.of("label", "output")) - .build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .labeledOutputTopics(Map.of("label", "output")) + .build())) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldWriteToLabeledOutputUsingProduced() { - final StreamsApp app = new SimpleApp() { - @Override - public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.toOutputTopic("label", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .labeledOutputTopics(Map.of("label", "output")) - .build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.toOutputTopic("label", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .labeledOutputTopics(Map.of("label", "output")) + .build())) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldWriteToError() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.toErrorTopic(); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .errorTopic("error") - .build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .errorTopic("error") + .build())) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldWriteToErrorUsingProduced() { - final StreamsApp app = new SimpleApp() { - @Override - public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.toErrorTopic(AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .errorTopic("error") - .build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.toErrorTopic(ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .errorTopic("error") + .build())) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldMap() { final KeyValueMapper> mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn(new KeyValue<>("baz", "qux")); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.map(mapper).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldMapNamed() { final KeyValueMapper> mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn(new KeyValue<>("baz", "qux")); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.map(mapper, Named.as("map")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldMapValues() { final ValueMapper mapper = mock(); when(mapper.apply("bar")).thenReturn("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.mapValues(mapper).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldMapValuesNamed() { final ValueMapper mapper = mock(); when(mapper.apply("bar")).thenReturn("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.mapValues(mapper, Named.as("map")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldMapValuesWithKey() { final ValueMapperWithKey mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.mapValues(mapper).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldMapValuesWithKeyNamed() { final ValueMapperWithKey mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.mapValues(mapper, Named.as("map")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldFlatMap() { final KeyValueMapper>> mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn(List.of(new KeyValue<>("baz", "qux"))); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.flatMap(mapper).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldFlatMapNamed() { final KeyValueMapper>> mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn(List.of(new KeyValue<>("baz", "qux"))); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.flatMap(mapper, Named.as("flatMap")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldFlatMapValues() { final ValueMapper> mapper = mock(); when(mapper.apply("bar")).thenReturn(List.of("baz")); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.flatMapValues(mapper).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldFlatMapValuesNamed() { final ValueMapper> mapper = mock(); when(mapper.apply("bar")).thenReturn(List.of("baz")); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.flatMapValues(mapper, Named.as("flatMap")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldFlatMapValuesWithKey() { final ValueMapperWithKey> mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn(List.of("baz")); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.flatMapValues(mapper).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test void shouldFlatMapValuesWithKeyNamed() { final ValueMapperWithKey> mapper = mock(); when(mapper.apply("foo", "bar")).thenReturn(List.of("baz")); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.flatMapValues(mapper, Named.as("flatMap")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test @@ -515,7 +482,7 @@ public void process(final Record inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -523,15 +490,14 @@ public void buildTopology(final TopologyBuilder builder) { processed.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test @@ -547,7 +513,7 @@ public void process(final Record inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -555,15 +521,14 @@ public void buildTopology(final TopologyBuilder builder) { processed.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test @@ -576,7 +541,7 @@ public void process(final Record inputRecord) { store.put(inputRecord.key(), inputRecord.value()); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() @@ -587,13 +552,12 @@ public void buildTopology(final TopologyBuilder builder) { input.process(processor, "my-store"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } } @Test @@ -606,7 +570,7 @@ public void process(final Record inputRecord) { store.put(inputRecord.key(), inputRecord.value()); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() @@ -617,13 +581,12 @@ public void buildTopology(final TopologyBuilder builder) { input.process(processor, Named.as("process"), "my-store"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } } @Test @@ -639,7 +602,7 @@ public void process(final FixedKeyRecord inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -647,15 +610,14 @@ public void buildTopology(final TopologyBuilder builder) { processed.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test @@ -671,7 +633,7 @@ public void process(final FixedKeyRecord inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -679,15 +641,14 @@ public void buildTopology(final TopologyBuilder builder) { processed.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } } @Test @@ -700,7 +661,7 @@ public void process(final FixedKeyRecord inputRecord) { store.put(inputRecord.key(), inputRecord.value()); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() @@ -711,13 +672,12 @@ public void buildTopology(final TopologyBuilder builder) { input.processValues(processor, "my-store"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } } @Test @@ -730,7 +690,7 @@ public void process(final FixedKeyRecord inputRecord) { store.put(inputRecord.key(), inputRecord.value()); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final StoreBuilder> store = builder.stores() @@ -741,13 +701,12 @@ public void buildTopology(final TopologyBuilder builder) { input.processValues(processor, Named.as("process"), "my-store"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - final KeyValueStore store = - topology.getTestDriver().getKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo")).isEqualTo("bar"); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } } @Test @@ -755,24 +714,23 @@ void shouldFilter() { final Predicate predicate = mock(); when(predicate.test("foo", "bar")).thenReturn(true); when(predicate.test("foo", "baz")).thenReturn(false); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.filter(predicate).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test @@ -780,24 +738,23 @@ void shouldFilterNamed() { final Predicate predicate = mock(); when(predicate.test("foo", "bar")).thenReturn(true); when(predicate.test("foo", "baz")).thenReturn(false); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.filter(predicate, Named.as("filter")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test @@ -805,24 +762,23 @@ void shouldFilterNot() { final Predicate predicate = mock(); when(predicate.test("foo", "bar")).thenReturn(false); when(predicate.test("foo", "baz")).thenReturn(true); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.filterNot(predicate).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test @@ -830,66 +786,63 @@ void shouldFilterNotNamed() { final Predicate predicate = mock(); when(predicate.test("foo", "bar")).thenReturn(false); when(predicate.test("foo", "baz")).thenReturn(true); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.filterNot(predicate, Named.as("filterNot")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldSelectKey() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.selectKey((k, v) -> v).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("bar") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldSelectKeyNamed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.selectKey((k, v) -> v, Named.as("select")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("bar") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test @@ -897,7 +850,7 @@ void shouldMapCapturingErrors() { final KeyValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -909,25 +862,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -935,7 +887,7 @@ void shouldMapCapturingErrorsNamed() { final KeyValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -947,25 +899,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -973,7 +924,7 @@ void shouldMapValuesCapturingErrors() { final ValueMapper mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); doReturn("success").when(mapper).apply("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -985,25 +936,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1011,7 +961,7 @@ void shouldMapValuesCapturingErrorsNamed() { final ValueMapper mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); doReturn("success").when(mapper).apply("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1023,25 +973,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1049,7 +998,7 @@ void shouldMapValuesWithKeyCapturingErrors() { final ValueMapperWithKey mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn("success").when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1061,25 +1010,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1087,7 +1035,7 @@ void shouldMapValuesWithKeyCapturingErrorsNamed() { final ValueMapperWithKey mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn("success").when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1099,25 +1047,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1125,7 +1072,7 @@ void shouldFlatMapCapturingErrors() { final KeyValueMapper>> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1137,25 +1084,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1163,7 +1109,7 @@ void shouldFlatMapCapturingErrorsNamed() { final KeyValueMapper>> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1175,25 +1121,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1201,7 +1146,7 @@ void shouldFlatMapValuesCapturingErrors() { final ValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); doReturn(List.of("success")).when(mapper).apply("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1213,25 +1158,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1239,7 +1183,7 @@ void shouldFlatMapValuesCapturingErrorsNamed() { final ValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); doReturn(List.of("success")).when(mapper).apply("baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1251,25 +1195,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1277,7 +1220,7 @@ void shouldFlatMapValuesWithKeyCapturingErrors() { final ValueMapperWithKey> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(List.of("success")).when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1289,25 +1232,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1315,7 +1257,7 @@ void shouldFlatMapValuesWithKeyCapturingErrorsNamed() { final ValueMapperWithKey> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(List.of("success")).when(mapper).apply("foo", "baz"); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1327,25 +1269,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1364,7 +1305,7 @@ public void process(final Record inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1376,25 +1317,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1413,7 +1353,7 @@ public void process(final Record inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1425,25 +1365,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1462,7 +1401,7 @@ public void process(final FixedKeyRecord inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1474,25 +1413,24 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test @@ -1511,7 +1449,7 @@ public void process(final FixedKeyRecord inputRecord) { throw new UnsupportedOperationException(); } }; - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1523,30 +1461,29 @@ public void buildTopology(final TopologyBuilder builder) { .to("error"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input().add("foo", "bar"); - topology.streamOutput("output") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.input().add("foo", "baz"); - topology.streamOutput("output") - .expectNextRecord() - .hasKey("foo") - .hasValue("success") - .expectNoMoreRecord(); - topology.streamOutput("error") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } } @Test void shouldRepartition() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1554,52 +1491,47 @@ public void buildTopology(final TopologyBuilder builder) { repartitioned.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldRepartitionUsingRepartitioned() { - final StreamsApp app = new SimpleApp() { - @Override - public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KStreamX repartitioned = input.repartition( - AutoRepartitioned.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - repartitioned.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX repartitioned = + input.repartition(RepartitionedX.with(Serdes.Long(), Serdes.Long())); + repartitioned.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldConvertToTable() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1607,21 +1539,20 @@ public void buildTopology(final TopologyBuilder builder) { table.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldConvertToTableNamed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1629,83 +1560,74 @@ public void buildTopology(final TopologyBuilder builder) { table.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldConvertToTableUsingMaterialized() { - final StreamsApp app = new SimpleApp() { - @Override - public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KTableX table = input.toTable( - AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - table.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX table = + input.toTable(MaterializedX.with(Serdes.Long(), Serdes.Long())); + table.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldConvertToTableNamedUsingMaterialized() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KTableX table = input.toTable(Named.as("toTable"), - AutoMaterialized.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - table.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + MaterializedX.with(Serdes.Long(), Serdes.Long())); + table.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldJoin() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1716,23 +1638,22 @@ public void buildTopology(final TopologyBuilder builder) { joined.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldJoinWithKey() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1743,23 +1664,22 @@ public void buildTopology(final TopologyBuilder builder) { joined.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldLeftJoin() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1770,30 +1690,29 @@ public void buildTopology(final TopologyBuilder builder) { joined.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("other_input") - .at(0L) - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("other_input") + .at(0L) + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldLeftJoinWithKey() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1804,30 +1723,29 @@ public void buildTopology(final TopologyBuilder builder) { joined.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - topology.input("other_input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("other_input") - .at(0L) - .add("foo", "baz"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("other_input") + .at(0L) + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldOuterJoin() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1838,30 +1756,29 @@ public void buildTopology(final TopologyBuilder builder) { joined.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("other_input") - .add("foo", "baz"); - topology.input("input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("input") - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("input") + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldOuterJoinWithKey() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -1872,306 +1789,275 @@ public void buildTopology(final TopologyBuilder builder) { joined.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("other_input") - .add("foo", "baz"); - topology.input("input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("input") - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("input") + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } } @Test void shouldJoinUsingJoined() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX otherInput = builder.stream("other_input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX joined = input.join(otherInput, Long::sum, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldJoinWithKeyUsingJoined() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX otherInput = builder.stream("other_input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX joined = input.join(otherInput, (k, v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldLeftJoinUsingJoined() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX otherInput = builder.stream("other_input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(0L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldLeftJoinWithKeyUsingJoined() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX otherInput = builder.stream("other_input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX joined = input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(0L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldOuterJoinUsingJoined() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX otherInput = builder.stream("other_input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(0L) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(3L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(3L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldOuterJoinWithKeyUsingJoined() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX otherInput = builder.stream("other_input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + ConsumedX.with(Serdes.Long(), Serdes.Long())); final KStreamX joined = input.outerJoin(otherInput, (k, v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - AutoStreamJoined.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()), Preconfigured.create(Serdes.Long()))); - joined.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); - topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .at(0L) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(3L) - .expectNextRecord() - .hasKey(1L) - .hasValue(5L) - .expectNoMoreRecord(); - topology.stop(); + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add(2L, 0L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .at(0L) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(3L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } } @Test void shouldGroupByKey() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -2180,62 +2066,57 @@ public void buildTopology(final TopologyBuilder builder) { count.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("foo", "baz"); - topology.streamOutput() - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey("foo") - .hasValue(1L) - .expectNextRecord() - .hasKey("foo") - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldGroupByKeyUsingGrouped() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupByKey( - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX count = grouped.count(); - count.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(1L) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + count.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldGroupBy() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); @@ -2244,62 +2125,57 @@ public void buildTopology(final TopologyBuilder builder) { count.toStream().to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .add("foo", "bar") - .add("baz", "bar"); - topology.streamOutput() - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey("bar") - .hasValue(1L) - .expectNextRecord() - .hasKey("bar") - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldGroupByUsingGrouped() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - final KGroupedStreamX grouped = input.groupBy((k, v) -> v, - AutoGrouped.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = + input.groupBy((k, v) -> v, GroupedX.with(Serdes.Long(), Serdes.Long())); final KTableX count = grouped.count(); - count.toStream().to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(2L) - .hasValue(1L) - .expectNextRecord() - .hasKey(2L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + count.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(2L) + .hasValue(1L) + .expectNextRecord() + .hasKey(2L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldMerge() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input1 = builder.stream("input1"); @@ -2308,26 +2184,25 @@ public void buildTopology(final TopologyBuilder builder) { grouped.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input1") - .add("foo", "bar"); - topology.input("input2") - .add("baz", "qux"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input1") + .add("foo", "bar"); + topology.input("input2") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldMergeNamed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input1 = builder.stream("input1"); @@ -2336,107 +2211,102 @@ public void buildTopology(final TopologyBuilder builder) { grouped.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input1") - .add("foo", "bar"); - topology.input("input2") - .add("baz", "qux"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNextRecord() - .hasKey("baz") - .hasValue("qux") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input1") + .add("foo", "bar"); + topology.input("input2") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } } @Test void shouldDoForEach() { final ForeachAction action = mock(); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.foreach(action); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - verify(action).apply("foo", "bar"); - verifyNoMoreInteractions(action); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + } } @Test void shouldDoForEachNamed() { final ForeachAction action = mock(); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.foreach(action, Named.as("forEach")); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - verify(action).apply("foo", "bar"); - verifyNoMoreInteractions(action); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + } } @Test void shouldPeek() { final ForeachAction action = mock(); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.peek(action).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - verify(action).apply("foo", "bar"); - verifyNoMoreInteractions(action); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + } } @Test void shouldPeekNamed() { final ForeachAction action = mock(); - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.peek(action, Named.as("peek")).to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder().build()); - topology.input("input") - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - verify(action).apply("foo", "bar"); - verifyNoMoreInteractions(action); - topology.stop(); + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + verify(action).apply("foo", "bar"); + verifyNoMoreInteractions(action); + } } private abstract static class SimpleProcessor implements Processor { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java new file mode 100644 index 00000000..f13e3509 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -0,0 +1,804 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Predicate; +import org.junit.jupiter.api.Test; + +class KTableXTest { + + @Test + void shouldConvertToStream() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + input.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldConvertToStreamNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + input.toStream(Named.as("toStream")).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldConvertToStreamMapping() { + final KeyValueMapper mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + input.toStream(mapper).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldConvertToStreamMappingNamed() { + final KeyValueMapper mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + input.toStream(mapper, Named.as("toStream")).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilter() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("baz", "qux")).thenReturn(false); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX filtered = input.filter(predicate); + filtered.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterNamed() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("baz", "qux")).thenReturn(false); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX filtered = input.filter(predicate, Named.as("filter")); + filtered.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterUsingMaterialized() { + final Predicate predicate = mock(); + when(predicate.test(1L, 2L)).thenReturn(true); + when(predicate.test(3L, 4L)).thenReturn(false); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX filtered = input.filter(predicate, + MaterializedX.with(Serdes.Long(), Serdes.Long())); + filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 4L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(3L) + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterNamedUsingMaterialized() { + final Predicate predicate = mock(); + when(predicate.test(1L, 2L)).thenReturn(true); + when(predicate.test(3L, 4L)).thenReturn(false); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX filtered = input.filter(predicate, Named.as("filter"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 4L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(3L) + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterNot() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("baz", "qux")).thenReturn(true); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX filtered = input.filterNot(predicate); + filtered.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterNotNamed() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("baz", "qux")).thenReturn(true); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX filtered = input.filterNot(predicate, Named.as("filter")); + filtered.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("baz") + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterNotUsingMaterialized() { + final Predicate predicate = mock(); + when(predicate.test(1L, 2L)).thenReturn(false); + when(predicate.test(3L, 4L)).thenReturn(true); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX filtered = input.filterNot(predicate, + MaterializedX.with(Serdes.Long(), Serdes.Long())); + filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 4L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(3L) + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldFilterNotNamedUsingMaterialized() { + final Predicate predicate = mock(); + when(predicate.test(1L, 2L)).thenReturn(false); + when(predicate.test(3L, 4L)).thenReturn(true); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX filtered = input.filterNot(predicate, Named.as("filter"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 4L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(3L) + .hasValue(null) + .expectNoMoreRecord(); + } + } + + @Test + void shouldGroupBy() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k)); + grouped.count().toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldGroupByUsingGrouped() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k), + GroupedX.with(Serdes.Long(), Serdes.Long())); + grouped.count().toStream() + .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L) + .add(3L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(2L) + .hasValue(1L) + .expectNextRecord() + .hasKey(2L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.join(otherInput, (v1, v2) -> v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.join(otherInput, (v1, v2) -> v1 + v2, Named.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldJoinUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX joined = input.join(otherInput, Long::sum, + MaterializedX.with(Serdes.Long(), Serdes.Long())); + joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldJoinNamedUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX joined = input.join(otherInput, Long::sum, Named.as("join"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2, Named.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftJoinUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, + MaterializedX.with(Serdes.Long(), Serdes.Long())); + joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftJoinNamedUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2, + Named.as("join"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldOuterJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.outerJoin(otherInput, + (v1, v2) -> v1 == null ? v2 : v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldOuterJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.outerJoin(otherInput, + (v1, v2) -> v1 == null ? v2 : v1 + v2, Named.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldOuterJoinUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, + MaterializedX.with(Serdes.Long(), Serdes.Long())); + joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(3L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldOuterJoinNamedUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX joined = input.outerJoin(otherInput, + (v1, v2) -> v1 == null ? v2 : v1 + v2, + Named.as("join"), + MaterializedX.with(Serdes.Long(), Serdes.Long())); + joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(3L) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java similarity index 79% rename from streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java rename to streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java index 2a01ba64..35066da8 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleApp.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java @@ -24,9 +24,18 @@ package com.bakdata.kafka; +import com.bakdata.fluent_kafka_streams_tests.TestTopology; import org.apache.kafka.common.serialization.Serdes.StringSerde; -abstract class SimpleApp implements StreamsApp { +abstract class StringApp implements StreamsApp { + + TestTopology startApp(final StreamsTopicConfig topicConfig) { + return TestHelper.startApp(this, topicConfig); + } + + TestTopology startApp() { + return TestHelper.startApp(this, StreamsTopicConfig.builder().build()); + } @Override public String getUniqueAppId(final StreamsTopicConfig topics) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java new file mode 100644 index 00000000..c919ddf1 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java @@ -0,0 +1,40 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import lombok.experimental.UtilityClass; + +@UtilityClass +class TestHelper { + static TestTopology startApp(final StreamsApp app, final StreamsTopicConfig topicConfig) { + final ConfiguredStreamsApp configuredStreamsApp = + new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig)); + final TestTopology topology = + TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + topology.start(); + return topology; + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index b01d4705..96c22dc6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -24,223 +24,433 @@ package com.bakdata.kafka; -import static com.bakdata.kafka.KStreamXTest.startApp; - import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.util.List; import java.util.Map; import java.util.regex.Pattern; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Materialized; import org.junit.jupiter.api.Test; class TopologyBuilderTest { @Test void shouldReadFromInput() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInput(); input.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .inputTopics(List.of("input")) - .build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .build())) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldReadFromInputUsingConsumed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.streamInput( - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); + final KStreamX input = + builder.streamInput(ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .inputTopics(List.of("input")) - .build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .inputTopics(List.of("input")) + .build())) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldReadFromLabeledInput() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInput("label"); input.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .labeledInputTopics(Map.of("label", List.of("input"))) - .build()); - topology.input().add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .labeledInputTopics(Map.of("label", List.of("input"))) + .build())) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldReadFromLabeledInputUsingConsumed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInput("label", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .labeledInputTopics(Map.of("label", List.of("input"))) - .build()); - topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .labeledInputTopics(Map.of("label", List.of("input"))) + .build())) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldReadFromPatternInput() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInputPattern(); input.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .inputPattern(Pattern.compile("input\\d+")) - .build()); - topology.input("input1").add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .inputPattern(Pattern.compile("input\\d+")) + .build())) { + topology.input("input1").add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldReadFromPatternInputUsingConsumed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInputPattern( - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .inputPattern(Pattern.compile("input\\d+")) - .build()); - topology.input("input1") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .inputPattern(Pattern.compile("input\\d+")) + .build())) { + topology.input("input1") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } @Test void shouldReadFromLabeledPatternInput() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInputPattern("label"); input.to("output"); } }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) - .build()); - topology.input("input1").add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - topology.stop(); + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) + .build())) { + topology.input("input1").add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } } @Test void shouldReadFromLabeledPatternInputUsingConsumed() { - final StreamsApp app = new SimpleApp() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.streamInputPattern("label", - AutoConsumed.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - input.to("output", AutoProduced.with(Preconfigured.create(Serdes.Long()), - Preconfigured.create(Serdes.Long()))); - } - }; - final TestTopology topology = - startApp(app, StreamsTopicConfig.builder() - .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) - .build()); - topology.input("input1") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); - topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .expectNextRecord() - .hasKey(1L) - .hasValue(2L) - .expectNoMoreRecord(); - topology.stop(); + ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) + .build())) { + topology.input("input1") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadFromTopic() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadFromTopicUsingConsumed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadFromTopics() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream(List.of("input")); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadFromTopicsUsingConsumed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream(List.of("input"), + ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadFromTopicPattern() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream(Pattern.compile("input\\d+")); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input1").add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadFromTopicPatternUsingConsumed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream(Pattern.compile("input\\d+"), + ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input1") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadTableFromTopic() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + input.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadTableFromTopicUsingConsumed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadTableFromTopicUsingMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input", + MaterializedX.with(Serdes.Long(), Serdes.Long())); + input.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadTableFromTopicUsingConsumedAndMaterialized() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long()), + Materialized.as("store")); + input.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java index 30dd0130..4e86cc5b 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/test_applications/MirrorWithNonDefaultSerde.java @@ -24,10 +24,10 @@ package com.bakdata.kafka.test_applications; -import com.bakdata.kafka.AutoConsumed; -import com.bakdata.kafka.AutoProduced; +import com.bakdata.kafka.ConsumedX; import com.bakdata.kafka.KStreamX; import com.bakdata.kafka.Preconfigured; +import com.bakdata.kafka.ProducedX; import com.bakdata.kafka.SerdeConfig; import com.bakdata.kafka.StreamsApp; import com.bakdata.kafka.StreamsTopicConfig; @@ -54,8 +54,8 @@ public void buildTopology(final TopologyBuilder builder) { final Preconfigured> valueSerde = newValueSerde(); final Preconfigured> keySerde = newKeySerde(); final KStreamX input = - builder.streamInput(AutoConsumed.with(keySerde, valueSerde)); - input.toOutputTopic(AutoProduced.with(keySerde, valueSerde)); + builder.streamInput(ConsumedX.with(keySerde, valueSerde)); + input.toOutputTopic(ProducedX.with(keySerde, valueSerde)); } @Override From 6bbbcc20933331f75b4f68f621484f9ce0cb1049 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 10:44:43 +0100 Subject: [PATCH 42/72] Update tests --- .../java/com/bakdata/kafka/DoubleApp.java | 49 ++ .../bakdata/kafka/KGroupedStreamXTest.java | 220 ++++---- .../java/com/bakdata/kafka/KStreamXTest.java | 486 +++++++++--------- .../java/com/bakdata/kafka/KTableXTest.java | 425 +++++++-------- .../bakdata/kafka/TopologyBuilderTest.java | 240 ++++----- 5 files changed, 735 insertions(+), 685 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java new file mode 100644 index 00000000..5841f1d1 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java @@ -0,0 +1,49 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import org.apache.kafka.common.serialization.Serdes.DoubleSerde; + +abstract class DoubleApp implements StreamsApp { + + @Override + public String getUniqueAppId(final StreamsTopicConfig topics) { + return "my-app"; + } + + @Override + public SerdeConfig defaultSerializationConfig() { + return new SerdeConfig(DoubleSerde.class, DoubleSerde.class); + } + + TestTopology startApp(final StreamsTopicConfig topicConfig) { + return TestHelper.startApp(this, topicConfig); + } + + TestTopology startApp() { + return TestHelper.startApp(this, StreamsTopicConfig.builder().build()); + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index 4b99d0de..498a8e43 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -87,32 +87,32 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldCountUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX counted = - grouped.count(MaterializedX.with(Serdes.Long(), Serdes.Long())); - counted.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX counted = + grouped.count(MaterializedX.keySerde(Serdes.String())); + counted.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) + .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long()) .expectNextRecord() - .hasKey(1L) + .hasKey("foo") .hasValue(1L) .expectNextRecord() - .hasKey(1L) + .hasKey("foo") .hasValue(2L) .expectNoMoreRecord(); } @@ -120,32 +120,32 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldCountNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX counted = grouped.count(Named.as("count"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - counted.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX counted = grouped.count(Named.as("count"), + MaterializedX.with(Serdes.String(), Serdes.Long())); + counted.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) + .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long()) .expectNextRecord() - .hasKey(1L) + .hasKey("foo") .hasValue(1L) .expectNextRecord() - .hasKey(1L) + .hasKey("foo") .hasValue(2L) .expectNoMoreRecord(); } @@ -179,66 +179,66 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReduceUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX reduced = grouped.reduce(Long::sum, - MaterializedX.with(Serdes.Long(), Serdes.Long())); - reduced.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, + MaterializedX.with(Serdes.String(), Serdes.String())); + reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldReduceNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX reduced = grouped.reduce(Long::sum, Named.as("reduce"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - reduced.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, + Named.as("reduce"), MaterializedX.with(Serdes.String(), Serdes.String())); + reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @@ -272,70 +272,70 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldAggregateUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX aggregated = - grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, - MaterializedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX aggregated = + grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value, + MaterializedX.with(Serdes.String(), Serdes.String())); aggregated.toStream() - .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + .to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldAggregateNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX aggregated = - grouped.aggregate(() -> 0L, (key, value, aggregate) -> aggregate + value, Named.as("aggregate"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX aggregated = + grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value, + Named.as("aggregate"), MaterializedX.with(Serdes.String(), Serdes.String())); aggregated.toStream() - .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + .to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 48da3bb6..be40c595 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -96,27 +96,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldWriteToOutputUsingProduced() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.toOutputTopic(ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.toOutputTopic(ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .outputTopic("output") .build())) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -144,27 +144,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldWriteToLabeledOutputUsingProduced() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.toOutputTopic("label", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.toOutputTopic("label", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .labeledOutputTopics(Map.of("label", "output")) .build())) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -192,27 +192,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldWriteToErrorUsingProduced() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.toErrorTopic(ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.toErrorTopic(ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .errorTopic("error") .build())) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -1504,27 +1504,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldRepartitionUsingRepartitioned() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX repartitioned = - input.repartition(RepartitionedX.with(Serdes.Long(), Serdes.Long())); - repartitioned.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX repartitioned = + input.repartition(RepartitionedX.with(Serdes.String(), Serdes.String())); + repartitioned.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -1573,54 +1573,54 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldConvertToTableUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX table = - input.toTable(MaterializedX.with(Serdes.Long(), Serdes.Long())); - table.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX table = + input.toTable(MaterializedX.with(Serdes.String(), Serdes.String())); + table.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @Test void shouldConvertToTableNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX table = input.toTable(Named.as("toTable"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - table.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX table = input.toTable(Named.as("toTable"), + MaterializedX.with(Serdes.String(), Serdes.String())); + table.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -1811,246 +1811,246 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldJoinUsingJoined() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX joined = input.join(otherInput, - Long::sum, + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, + (v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); - joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldJoinWithKeyUsingJoined() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX joined = input.join(otherInput, + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, (k, v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); - joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldLeftJoinUsingJoined() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX joined = input.leftJoin(otherInput, + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); - joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); + .add("bar", ""); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(0L) - .add(1L, 3L); + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldLeftJoinWithKeyUsingJoined() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX joined = input.leftJoin(otherInput, + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); - joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); + .add("bar", ""); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(0L) - .add(1L, 3L); + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldOuterJoinUsingJoined() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX joined = input.outerJoin(otherInput, + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); - joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); + .add("bar", ""); topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(0L) - .add(1L, 2L); + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(3L) + .hasKey("foo") + .hasValue("baz") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldOuterJoinWithKeyUsingJoined() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KStreamX joined = input.outerJoin(otherInput, + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.outerJoin(otherInput, (k, v1, v2) -> v1 == null ? v2 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); - joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add(2L, 0L); + .add("bar", ""); topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .at(0L) - .add(1L, 2L); + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(3L) + .hasKey("foo") + .hasValue("baz") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @@ -2084,31 +2084,31 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldGroupByKeyUsingGrouped() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX count = grouped.count(); - count.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX count = grouped.count(); + count.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) + .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long()) .expectNextRecord() - .hasKey(1L) + .hasKey("foo") .hasValue(1L) .expectNextRecord() - .hasKey(1L) + .hasKey("foo") .hasValue(2L) .expectNoMoreRecord(); } @@ -2143,31 +2143,31 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldGroupByUsingGrouped() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedStreamX grouped = - input.groupBy((k, v) -> v, GroupedX.with(Serdes.Long(), Serdes.Long())); - final KTableX count = grouped.count(); - count.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupBy((k, v) -> v, GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX count = grouped.count(); + count.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) + .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long()) .expectNextRecord() - .hasKey(2L) + .hasKey("bar") .hasValue(1L) .expectNextRecord() - .hasKey(2L) + .hasKey("bar") .hasValue(2L) .expectNoMoreRecord(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index f13e3509..0a5b812e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -179,33 +179,33 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldFilterUsingMaterialized() { - final Predicate predicate = mock(); - when(predicate.test(1L, 2L)).thenReturn(true); - when(predicate.test(3L, 4L)).thenReturn(false); - final StringApp app = new StringApp() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("baz", "qux")).thenReturn(false); + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX filtered = input.filter(predicate, - MaterializedX.with(Serdes.Long(), Serdes.Long())); - filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX filtered = input.filter(predicate, + MaterializedX.with(Serdes.String(), Serdes.String())); + filtered.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 4L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "qux"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(3L) + .hasKey("baz") .hasValue(null) .expectNoMoreRecord(); } @@ -213,33 +213,33 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldFilterNamedUsingMaterialized() { - final Predicate predicate = mock(); - when(predicate.test(1L, 2L)).thenReturn(true); - when(predicate.test(3L, 4L)).thenReturn(false); - final StringApp app = new StringApp() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(true); + when(predicate.test("baz", "qux")).thenReturn(false); + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX filtered = input.filter(predicate, Named.as("filter"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX filtered = input.filter(predicate, Named.as("filter"), + MaterializedX.with(Serdes.String(), Serdes.String())); + filtered.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 4L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "qux"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(3L) + .hasKey("baz") .hasValue(null) .expectNoMoreRecord(); } @@ -303,33 +303,33 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldFilterNotUsingMaterialized() { - final Predicate predicate = mock(); - when(predicate.test(1L, 2L)).thenReturn(false); - when(predicate.test(3L, 4L)).thenReturn(true); - final StringApp app = new StringApp() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("baz", "qux")).thenReturn(true); + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX filtered = input.filterNot(predicate, - MaterializedX.with(Serdes.Long(), Serdes.Long())); - filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX filtered = input.filterNot(predicate, + MaterializedX.with(Serdes.String(), Serdes.String())); + filtered.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 4L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "qux"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(3L) + .hasKey("baz") .hasValue(null) .expectNoMoreRecord(); } @@ -337,33 +337,33 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldFilterNotNamedUsingMaterialized() { - final Predicate predicate = mock(); - when(predicate.test(1L, 2L)).thenReturn(false); - when(predicate.test(3L, 4L)).thenReturn(true); - final StringApp app = new StringApp() { + final Predicate predicate = mock(); + when(predicate.test("foo", "bar")).thenReturn(false); + when(predicate.test("baz", "qux")).thenReturn(true); + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX filtered = input.filterNot(predicate, Named.as("filter"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - filtered.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX filtered = input.filterNot(predicate, Named.as("filter"), + MaterializedX.with(Serdes.String(), Serdes.String())); + filtered.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 4L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "qux"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(3L) + .hasKey("baz") .hasValue(null) .expectNoMoreRecord(); } @@ -398,31 +398,31 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldGroupByUsingGrouped() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k), - GroupedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k), + GroupedX.with(Serdes.String(), Serdes.String())); grouped.count().toStream() - .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + .to("output", ProducedX.with(Serdes.String(), Serdes.Long())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L) - .add(3L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) + .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long()) .expectNextRecord() - .hasKey(2L) + .hasKey("bar") .hasValue(1L) .expectNextRecord() - .hasKey(2L) + .hasKey("bar") .hasValue(2L) .expectNoMoreRecord(); } @@ -478,66 +478,66 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldJoinUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX otherInput = builder.table("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX joined = input.join(otherInput, Long::sum, - MaterializedX.with(Serdes.Long(), Serdes.Long())); - joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.join(otherInput, (v1, v2) -> v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldJoinNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX otherInput = builder.table("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX joined = input.join(otherInput, Long::sum, Named.as("join"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.join(otherInput, (v1, v2) -> v1 + v2, Named.as("join"), + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @@ -600,74 +600,74 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldLeftJoinUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX otherInput = builder.table("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, - MaterializedX.with(Serdes.Long(), Serdes.Long())); - joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldLeftJoinNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX otherInput = builder.table("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX joined = input.leftJoin(otherInput, + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, Named.as("join"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @@ -730,74 +730,75 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldOuterJoinUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX otherInput = builder.table("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, - MaterializedX.with(Serdes.Long(), Serdes.Long())); - joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(3L) + .hasKey("foo") + .hasValue("baz") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } @Test void shouldOuterJoinNamedUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX otherInput = builder.table("other_input", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - final KTableX joined = input.outerJoin(otherInput, + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = builder.table("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.outerJoin(otherInput, (v1, v2) -> v1 == null ? v2 : v1 + v2, Named.as("join"), - MaterializedX.with(Serdes.Long(), Serdes.Long())); - joined.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("other_input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 3L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); topology.input("input") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(3L) + .hasKey("foo") + .hasValue("baz") .expectNextRecord() - .hasKey(1L) - .hasValue(5L) + .hasKey("foo") + .hasValue("barbaz") .expectNoMoreRecord(); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index 96c22dc6..87d66509 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -57,27 +57,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromInputUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.streamInput(ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.streamInput(ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .inputTopics(List.of("input")) .build())) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -105,27 +105,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromLabeledInputUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.streamInput("label", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = builder.streamInput("label", + ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .labeledInputTopics(Map.of("label", List.of("input"))) .build())) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -153,27 +153,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromPatternInputUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.streamInputPattern( - ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = builder.streamInputPattern( + ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .inputPattern(Pattern.compile("input\\d+")) .build())) { topology.input("input1") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -201,27 +201,27 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromLabeledPatternInputUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.streamInputPattern("label", - ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = builder.streamInputPattern("label", + ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() .labeledInputPatterns(Map.of("label", Pattern.compile("input\\d+"))) .build())) { topology.input("input1") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -247,25 +247,25 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromTopicUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -291,25 +291,25 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromTopicsUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream(List.of("input"), - ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = builder.stream(List.of("input"), + ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -335,25 +335,25 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadFromTopicPatternUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream(Pattern.compile("input\\d+"), - ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX input = builder.stream(Pattern.compile("input\\d+"), + ConsumedX.with(Serdes.String(), Serdes.String())); + input.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input1") - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @@ -379,76 +379,76 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldReadTableFromTopicUsingConsumed() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); - input.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @Test void shouldReadTableFromTopicUsingMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = builder.table("input", - MaterializedX.with(Serdes.Long(), Serdes.Long())); - input.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + final KTableX input = builder.table("input", + MaterializedX.with(Serdes.String(), Serdes.String())); + input.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } @Test void shouldReadTableFromTopicUsingConsumedAndMaterialized() { - final StringApp app = new StringApp() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KTableX input = - builder.table("input", ConsumedX.with(Serdes.Long(), Serdes.Long()), + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String()), Materialized.as("store")); - input.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + input.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) - .add(1L, 2L); + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.Long()) - .withValueSerde(Serdes.Long()) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() - .hasKey(1L) - .hasValue(2L) + .hasKey("foo") + .hasValue("bar") .expectNoMoreRecord(); } } From 3201fce58e7192490f76c603fd06b6aad9015177 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 11:17:05 +0100 Subject: [PATCH 43/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 541 ++++++++++++++---- .../java/com/bakdata/kafka/KTableXTest.java | 210 +++++++ 2 files changed, 630 insertions(+), 121 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index be40c595..ecccf7be 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -37,6 +37,7 @@ import java.util.Map; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Branched; import org.apache.kafka.streams.kstream.ForeachAction; import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.KeyValueMapper; @@ -1652,24 +1653,33 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldJoinWithKey() { - final StringApp app = new StringApp() { + void shouldJoinUsingJoined() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input"); - final KStreamX otherInput = builder.stream("other_input"); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); final KStreamX joined = input.join(otherInput, - (k, v1, v2) -> v1 + v2, - JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); - joined.to("output"); + (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .add("foo", "bar"); topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .add("foo", "baz"); topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() .hasKey("foo") .hasValue("barbaz") @@ -1678,14 +1688,14 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldLeftJoin() { + void shouldJoinWithKey() { final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KStreamX otherInput = builder.stream("other_input"); - final KStreamX joined = input.leftJoin(otherInput, - (v1, v2) -> v2 == null ? v1 : v1 + v2, + final KStreamX joined = input.join(otherInput, + (k, v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); } @@ -1694,15 +1704,8 @@ public void buildTopology(final TopologyBuilder builder) { topology.input("input") .add("foo", "bar"); topology.input("other_input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("other_input") - .at(0L) .add("foo", "baz"); topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") .expectNextRecord() .hasKey("foo") .hasValue("barbaz") @@ -1711,31 +1714,33 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldLeftJoinWithKey() { - final StringApp app = new StringApp() { + void shouldJoinWithKeyUsingJoined() { + final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input"); - final KStreamX otherInput = builder.stream("other_input"); - final KStreamX joined = input.leftJoin(otherInput, - (k, v1, v2) -> v2 == null ? v1 : v1 + v2, - JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); - joined.to("output"); + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX otherInput = builder.stream("other_input", + ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, + (k, v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .add("foo", "bar"); topology.input("other_input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("other_input") - .at(0L) + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .add("foo", "baz"); topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) .expectNextRecord() .hasKey("foo") .hasValue("barbaz") @@ -1744,64 +1749,31 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldOuterJoin() { + void shouldLeftJoin() { final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KStreamX otherInput = builder.stream("other_input"); - final KStreamX joined = input.outerJoin(otherInput, - (v1, v2) -> v1 == null ? v2 : v1 + v2, + final KStreamX joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); joined.to("output"); } }; try (final TestTopology topology = app.startApp()) { - topology.input("other_input") - .add("foo", "baz"); topology.input("input") - .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush - .add("bar", ""); - topology.input("input") - .at(0L) .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("baz") - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - } - } - - @Test - void shouldOuterJoinWithKey() { - final StringApp app = new StringApp() { - @Override - public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = builder.stream("input"); - final KStreamX otherInput = builder.stream("other_input"); - final KStreamX joined = input.outerJoin(otherInput, - (k, v1, v2) -> v1 == null ? v2 : v1 + v2, - JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); - joined.to("output"); - } - }; - try (final TestTopology topology = app.startApp()) { topology.input("other_input") - .add("foo", "baz"); - topology.input("input") .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush .add("bar", ""); - topology.input("input") + topology.input("other_input") .at(0L) - .add("foo", "bar"); + .add("foo", "baz"); topology.streamOutput() .expectNextRecord() .hasKey("foo") - .hasValue("baz") + .hasValue("bar") .expectNextRecord() .hasKey("foo") .hasValue("barbaz") @@ -1810,7 +1782,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldJoinUsingJoined() { + void shouldLeftJoinUsingJoined() { final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -1818,8 +1790,8 @@ public void buildTopology(final TopologyBuilder builder) { builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); final KStreamX otherInput = builder.stream("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX joined = input.join(otherInput, - (v1, v2) -> v1 + v2, + final KStreamX joined = input.leftJoin(otherInput, + (v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); @@ -1833,45 +1805,52 @@ public void buildTopology(final TopologyBuilder builder) { topology.input("other_input") .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .at(0L) .add("foo", "baz"); topology.streamOutput() .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String()) .expectNextRecord() .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") .hasValue("barbaz") .expectNoMoreRecord(); } } @Test - void shouldJoinWithKeyUsingJoined() { - final DoubleApp app = new DoubleApp() { + void shouldLeftJoinWithKey() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX joined = input.join(otherInput, - (k, v1, v2) -> v1 + v2, - JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); - joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.leftJoin(otherInput, + (k, v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); } }; - try (final TestTopology topology = app.startApp()) { + try (final TestTopology topology = app.startApp()) { topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) .add("foo", "bar"); topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("other_input") + .at(0L) .add("foo", "baz"); topology.streamOutput() - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") .expectNextRecord() .hasKey("foo") .hasValue("barbaz") @@ -1880,7 +1859,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldLeftJoinUsingJoined() { + void shouldLeftJoinWithKeyUsingJoined() { final DoubleApp app = new DoubleApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -1889,7 +1868,7 @@ public void buildTopology(final TopologyBuilder builder) { final KStreamX otherInput = builder.stream("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); final KStreamX joined = input.leftJoin(otherInput, - (v1, v2) -> v2 == null ? v1 : v1 + v2, + (k, v1, v2) -> v2 == null ? v1 : v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); @@ -1924,42 +1903,31 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldLeftJoinWithKeyUsingJoined() { - final DoubleApp app = new DoubleApp() { + void shouldOuterJoin() { + final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { - final KStreamX input = - builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX otherInput = builder.stream("other_input", - ConsumedX.with(Serdes.String(), Serdes.String())); - final KStreamX joined = input.leftJoin(otherInput, - (k, v1, v2) -> v2 == null ? v1 : v1 + v2, - JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); - joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.outerJoin(otherInput, + (v1, v2) -> v1 == null ? v2 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) - .add("foo", "bar"); + try (final TestTopology topology = app.startApp()) { topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush .add("bar", ""); - topology.input("other_input") - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) + topology.input("input") .at(0L) - .add("foo", "baz"); + .add("foo", "bar"); topology.streamOutput() - .withKeySerde(Serdes.String()) - .withValueSerde(Serdes.String()) .expectNextRecord() .hasKey("foo") - .hasValue("bar") + .hasValue("baz") .expectNextRecord() .hasKey("foo") .hasValue("barbaz") @@ -2011,6 +1979,39 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldOuterJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = input.outerJoin(otherInput, + (k, v1, v2) -> v1 == null ? v2 : v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L))); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .at(Duration.ofMinutes(1L).toMillis() + 1) // trigger flush + .add("bar", ""); + topology.input("input") + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + @Test void shouldOuterJoinWithKeyUsingJoined() { final DoubleApp app = new DoubleApp() { @@ -2309,6 +2310,304 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldSplit() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final BranchedKStreamX branches = input.split(); + branches.defaultBranch(Branched.withConsumer(s -> s.to("output"))); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldSplitNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final BranchedKStreamX branches = input.split(Named.as("split")); + branches.defaultBranch(Branched.withConsumer(s -> s.to("output"))); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KTableX otherInput = builder.table("table_input"); + final KStreamX joined = input.join(otherInput, (v1, v2) -> v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableJoinUsingJoined() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("table_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, (v1, v2) -> v1 + v2, + JoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KTableX otherInput = builder.table("table_input"); + final KStreamX joined = input.join(otherInput, (k, v1, v2) -> v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableJoinWithKeyUsingJoined() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("table_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, (k, v1, v2) -> v1 + v2, + JoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableLeftJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KTableX otherInput = builder.table("table_input"); + final KStreamX joined = + input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableLeftJoinUsingJoined() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("table_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = + input.leftJoin(otherInput, (v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableLeftJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KTableX otherInput = builder.table("table_input"); + final KStreamX joined = + input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTableLeftJoinWithKeyUsingJoined() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("table_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = + input.leftJoin(otherInput, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, + JoinedX.with(Serdes.String(), Serdes.String(), Serdes.String())); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 0a5b812e..acce188b 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -33,6 +33,8 @@ import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.ValueMapper; +import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.junit.jupiter.api.Test; class KTableXTest { @@ -802,4 +804,212 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); } } + + @Test + void shouldMapValues() { + final ValueMapper mapper = mock(); + when(mapper.apply("bar")).thenReturn("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX mapped = input.mapValues(mapper); + mapped.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesNamed() { + final ValueMapper mapper = mock(); + when(mapper.apply("bar")).thenReturn("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX mapped = input.mapValues(mapper, Named.as("map")); + mapped.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesUsingMaterialized() { + final ValueMapper mapper = mock(); + when(mapper.apply("bar")).thenReturn("baz"); + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX mapped = + input.mapValues(mapper, MaterializedX.with(Serdes.String(), Serdes.String())); + mapped.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesNamedUsingMaterialized() { + final ValueMapper mapper = mock(); + when(mapper.apply("bar")).thenReturn("baz"); + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX mapped = input.mapValues(mapper, Named.as("mapped"), + MaterializedX.with(Serdes.String(), Serdes.String())); + mapped.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesWithKey() { + final ValueMapperWithKey mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX mapped = input.mapValues(mapper); + mapped.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesWithKeyNamed() { + final ValueMapperWithKey mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX mapped = input.mapValues(mapper, Named.as("map")); + mapped.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesWithKeyUsingMaterialized() { + final ValueMapperWithKey mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX mapped = + input.mapValues(mapper, MaterializedX.with(Serdes.String(), Serdes.String())); + mapped.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesWithKeyNamedUsingMaterialized() { + final ValueMapperWithKey mapper = mock(); + when(mapper.apply("foo", "bar")).thenReturn("baz"); + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX mapped = input.mapValues(mapper, Named.as("mapped"), + MaterializedX.with(Serdes.String(), Serdes.String())); + mapped.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } } From 8aaeac41a67c3c1802d11a0571050414dfda29de Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 12:19:32 +0100 Subject: [PATCH 44/72] Add tests --- .../bakdata/kafka/KGroupedStreamXTest.java | 129 ++++++++++++++++++ .../java/com/bakdata/kafka/KStreamXTest.java | 5 + .../java/com/bakdata/kafka/KTableXTest.java | 3 + 3 files changed, 137 insertions(+) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index 498a8e43..d1c85a2f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -25,8 +25,14 @@ package com.bakdata.kafka; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.SlidingWindows; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.kstream.Windowed; import org.junit.jupiter.api.Test; class KGroupedStreamXTest { @@ -339,4 +345,127 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); } } + + @Test + void shouldTimeWindow() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, Long> counted = windowed.count(); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:60000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldSlidingWindow() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(SlidingWindows.ofTimeDifferenceWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = windowed.count(); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:1") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30001") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30000") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldSessionWindow() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = windowed.count(); + final KStreamX output = counted.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", Produced.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(Duration.ofSeconds(30L) + .toMillis()) // if time is lower than session size, there is a bug in Kafka Streams + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index ecccf7be..9986e981 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -2608,6 +2608,11 @@ public void buildTopology(final TopologyBuilder builder) { } } + //TODO TopicNameExtractor + //TODO errorFilter + //TODO GlobalKTable joins + //TODO printed + private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index acce188b..2b2f0e5d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -1012,4 +1012,7 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); } } + + //TODO transformValues + //TODO FKey Join } From 67e239056e026899f194337fdfba69ea01269d6d Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 13:07:50 +0100 Subject: [PATCH 45/72] Add tests --- .../bakdata/kafka/KGroupedStreamXTest.java | 35 +- .../java/com/bakdata/kafka/KStreamXTest.java | 305 +++++++++++++++++- 2 files changed, 333 insertions(+), 7 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index d1c85a2f..583b4d71 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -28,7 +28,6 @@ import java.time.Duration; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.kstream.Named; -import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.SessionWindows; import org.apache.kafka.streams.kstream.SlidingWindows; import org.apache.kafka.streams.kstream.TimeWindows; @@ -439,13 +438,13 @@ public void buildTopology(final TopologyBuilder builder) { final KStreamX output = counted.toStream( (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() .toEpochMilli()); - output.to("output", Produced.valueSerde(Serdes.Long())); + output.to("output"); } }; try (final TestTopology topology = app.startApp()) { topology.input() - .at(Duration.ofSeconds(30L) - .toMillis()) // if time is lower than session size, there is a bug in Kafka Streams + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) .add("foo", "bar") .at(Duration.ofSeconds(60L).toMillis()) .add("foo", "baz") @@ -468,4 +467,32 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); } } + + @Test + void shouldCogroup() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final KTableX aggregated = cogrouped.aggregate(() -> ""); + aggregated.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 9986e981..b961685a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -32,17 +32,23 @@ import static org.mockito.Mockito.when; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.io.File; import java.time.Duration; import java.util.List; import java.util.Map; +import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.TestOutputTopic; import org.apache.kafka.streams.kstream.Branched; import org.apache.kafka.streams.kstream.ForeachAction; +import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.JoinWindows; import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Printed; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.processor.StateStore; @@ -62,6 +68,7 @@ import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; @@ -218,6 +225,63 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldWriteToUsingExtractor() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.to((key, value, recordContext) -> key); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input().add("foo", "bar"); + final StreamsConfig streamsConfig = topology.getStreamsConfig(); + final Serde keySerde = (Serde) streamsConfig.defaultKeySerde(); + final Serde valueSerde = (Serde) streamsConfig.defaultValueSerde(); + final TestOutputTopic outputTopic = topology.getTestDriver() + .createOutputTopic("foo", keySerde.deserializer(), valueSerde.deserializer()); + this.softly.assertThat(outputTopic.readRecordsToList()) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.getKey()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.getValue()).isEqualTo("bar"); + }); + } + } + + @Test + void shouldWriteToUsingExtractorAndProduced() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.to((key, value, recordContext) -> key, ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final Serde keySerde = Serdes.String(); + final Serde valueSerde = Serdes.String(); + final TestOutputTopic outputTopic = topology.getTestDriver() + .createOutputTopic("foo", keySerde.deserializer(), valueSerde.deserializer()); + this.softly.assertThat(outputTopic.readRecordsToList()) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.getKey()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.getValue()).isEqualTo("bar"); + }); + } + } + @Test void shouldMap() { final KeyValueMapper> mapper = mock(); @@ -2608,10 +2672,245 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO TopicNameExtractor + @Test + void shouldPrint(@TempDir final File tempDir) { + final File file = new File(tempDir, "out.txt"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.as("read")); + input.print(Printed.toFile(file.getAbsolutePath())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + } + this.softly.assertThat(file) + .exists() + .hasContent("[read]: foo, bar"); + } + + @Test + void shouldGlobalTableJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = + input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2, Named.as("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = input.join(otherInput, (k, v) -> k, (k, v1, v2) -> v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableJoinWithKeyNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = + input.join(otherInput, (k, v) -> k, (k, v1, v2) -> v1 + v2, Named.as("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableLeftJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = + input.leftJoin(otherInput, (k, v) -> k, (v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableLeftJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = + input.leftJoin(otherInput, (k, v) -> k, (v1, v2) -> v2 == null ? v1 : v1 + v2, + Named.as("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableLeftJoinWithKey() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = + input.leftJoin(otherInput, (k, v) -> k, (k, v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGlobalTableLeftJoinWithKeyNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = + input.leftJoin(otherInput, (k, v) -> k, (k, v1, v2) -> v2 == null ? v1 : v1 + v2, + Named.as("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("quxbaz") + .expectNoMoreRecord(); + } + } + //TODO errorFilter - //TODO GlobalKTable joins - //TODO printed private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; From 23c382cac7829fff2e5f167b80726a6496fe5b97 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 13:34:43 +0100 Subject: [PATCH 46/72] Add tests --- .../java/com/bakdata/kafka/KTableXTest.java | 316 +++++++++++++++++- 1 file changed, 315 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 2b2f0e5d..53ab0483 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -35,10 +35,25 @@ import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +@ExtendWith(SoftAssertionsExtension.class) class KTableXTest { + @InjectSoftAssertions + private SoftAssertions softly; + @Test void shouldConvertToStream() { final StringApp app = new StringApp() { @@ -1013,6 +1028,305 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO transformValues + @Test + void shouldTransformValues() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + if ("foo".equals(readOnlyKey) && "bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX transformed = input.transformValues(transformer); + transformed.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesNamed() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + if ("foo".equals(readOnlyKey) && "bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX transformed = input.transformValues(transformer, Named.as("transform")); + transformed.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesUsingMaterialized() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + if ("foo".equals(readOnlyKey) && "bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX transformed = + input.transformValues(transformer, MaterializedX.with(Serdes.String(), Serdes.String())); + transformed.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesNamedUsingMaterialized() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + if ("foo".equals(readOnlyKey) && "bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX transformed = + input.transformValues(transformer, MaterializedX.with(Serdes.String(), Serdes.String()), + Named.as("transform")); + transformed.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesUsingStore() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(readOnlyKey, value); + return value; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KTableX input = builder.table("input"); + final KTableX transformed = input.transformValues(transformer, "my-store"); + transformed.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValuesNamedUsingStore() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(readOnlyKey, value); + return value; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KTableX input = builder.table("input"); + final KTableX transformed = + input.transformValues(transformer, Named.as("transform"), "my-store"); + transformed.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValuesUsingStoreAndMaterialized() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(readOnlyKey, value); + return value; + } + }; + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Serdes.String(), + Serdes.String()); + builder.addStateStore(store); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX transformed = + input.transformValues(transformer, MaterializedX.with(Serdes.String(), Serdes.String()), + "my-store"); + transformed.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValuesNamedUsingStoreAndMaterialized() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String readOnlyKey, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(readOnlyKey, value); + return value; + } + }; + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Serdes.String(), + Serdes.String()); + builder.addStateStore(store); + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX transformed = + input.transformValues(transformer, MaterializedX.with(Serdes.String(), Serdes.String()), + Named.as("transform"), "my-store"); + transformed.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + //TODO FKey Join + + private abstract static class SimpleValueTransformer + implements ValueTransformerWithKey { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + @Override + public void close() { + // do nothing + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + + } } From 0e4e9df7402cb03cd513db93f8fc8ec5e9733445 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 14:08:23 +0100 Subject: [PATCH 47/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 833 ++++++++++++++++-- .../bakdata/kafka/TopologyBuilderTest.java | 3 + 2 files changed, 770 insertions(+), 66 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index b961685a..c36a8033 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -31,6 +31,7 @@ import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; +import com.bakdata.fluent_kafka_streams_tests.TestInput; import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.io.File; import java.time.Duration; @@ -41,6 +42,7 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.streams.TestOutputTopic; +import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.kstream.Branched; import org.apache.kafka.streams.kstream.ForeachAction; import org.apache.kafka.streams.kstream.GlobalKTable; @@ -947,6 +949,46 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldMapCapturingErrorsWithFilter() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage())); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + @Test void shouldMapCapturingErrorsNamed() { final KeyValueMapper> mapper = mock(); @@ -985,16 +1027,482 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldMapValuesCapturingErrors() { - final ValueMapper mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); - doReturn("success").when(mapper).apply("baz"); + void shouldMapCapturingErrorsNamedWithFilter() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage()), Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + + @Test + void shouldMapValuesCapturingErrors() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesCapturingErrorsWithFilter() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage())); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + + @Test + void shouldMapValuesCapturingErrorsNamed() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesCapturingErrorsNamedWithFilter() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage()), + Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + + @Test + void shouldMapValuesWithKeyCapturingErrors() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesWithKeyCapturingErrorsWithFilter() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage())); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + + @Test + void shouldMapValuesWithKeyCapturingErrorsNamed() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldMapValuesWithKeyCapturingErrorsNamedWithFilter() { + final ValueMapperWithKey mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn("success").when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage()), + Named.as("map")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + + @Test + void shouldFlatMapCapturingErrors() { + final KeyValueMapper>> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.flatMapCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatMapCapturingErrorsWithFilter() { + final KeyValueMapper>> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.flatMapCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage())); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + + @Test + void shouldFlatMapCapturingErrorsNamed() { + final KeyValueMapper>> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.flatMapCapturingErrors(mapper, Named.as("flatMap")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatMapCapturingErrorsNamedWithFilter() { + final KeyValueMapper>> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapValuesCapturingErrors(mapper); + input.flatMapCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage()), + Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1002,7 +1510,8 @@ public void buildTopology(final TopologyBuilder builder) { } }; try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); + final TestInput input = topology.input(); + input.add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); topology.streamOutput("error") @@ -1010,28 +1519,29 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - topology.input().add("foo", "baz"); + input.add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() - .hasKey("foo") - .hasValue("success") + .hasKey("success_key") + .hasValue("success_value") .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); } } @Test - void shouldMapValuesCapturingErrorsNamed() { - final ValueMapper mapper = mock(); + void shouldFlatMapValuesCapturingErrors() { + final ValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); - doReturn("success").when(mapper).apply("baz"); + doReturn(List.of("success")).when(mapper).apply("baz"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapValuesCapturingErrors(mapper, Named.as("map")); + input.flatMapValuesCapturingErrors(mapper); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1059,16 +1569,17 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldMapValuesWithKeyCapturingErrors() { - final ValueMapperWithKey mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn("success").when(mapper).apply("foo", "baz"); + void shouldFlatMapValuesCapturingErrorsWithFilter() { + final ValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn(List.of("success")).when(mapper).apply("baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("qux"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapValuesCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage())); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1076,7 +1587,8 @@ public void buildTopology(final TopologyBuilder builder) { } }; try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); + final TestInput input = topology.input(); + input.add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); topology.streamOutput("error") @@ -1084,7 +1596,7 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - topology.input().add("foo", "baz"); + input.add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() .hasKey("foo") @@ -1092,20 +1604,21 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); } } @Test - void shouldMapValuesWithKeyCapturingErrorsNamed() { - final ValueMapperWithKey mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn("success").when(mapper).apply("foo", "baz"); + void shouldFlatMapValuesCapturingErrorsNamed() { + final ValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn(List.of("success")).when(mapper).apply("baz"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.mapValuesCapturingErrors(mapper, Named.as("map")); + input.flatMapValuesCapturingErrors(mapper, Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1133,16 +1646,18 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFlatMapCapturingErrors() { - final KeyValueMapper>> mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + void shouldFlatMapValuesCapturingErrorsNamedWithFilter() { + final ValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn(List.of("success")).when(mapper).apply("baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("qux"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage()), + Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1150,7 +1665,8 @@ public void buildTopology(final TopologyBuilder builder) { } }; try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); + final TestInput input = topology.input(); + input.add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); topology.streamOutput("error") @@ -1158,28 +1674,29 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - topology.input().add("foo", "baz"); + input.add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") + .hasKey("foo") + .hasValue("success") .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); } } @Test - void shouldFlatMapCapturingErrorsNamed() { - final KeyValueMapper>> mapper = mock(); + void shouldFlatMapValuesWithKeyCapturingErrors() { + final ValueMapperWithKey> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn(List.of(KeyValue.pair("success_key", "success_value"))).when(mapper).apply("foo", "baz"); + doReturn(List.of("success")).when(mapper).apply("foo", "baz"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapCapturingErrors(mapper, Named.as("flatMap")); + input.flatMapValuesCapturingErrors(mapper); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1198,8 +1715,8 @@ public void buildTopology(final TopologyBuilder builder) { topology.input().add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() - .hasKey("success_key") - .hasValue("success_value") + .hasKey("foo") + .hasValue("success") .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); @@ -1207,16 +1724,17 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFlatMapValuesCapturingErrors() { - final ValueMapper> mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); - doReturn(List.of("success")).when(mapper).apply("baz"); + void shouldFlatMapValuesWithKeyCapturingErrorsWithFilter() { + final ValueMapperWithKey> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of("success")).when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapValuesCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage())); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1224,7 +1742,8 @@ public void buildTopology(final TopologyBuilder builder) { } }; try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); + final TestInput input = topology.input(); + input.add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); topology.streamOutput("error") @@ -1232,7 +1751,7 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - topology.input().add("foo", "baz"); + input.add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() .hasKey("foo") @@ -1240,14 +1759,15 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); } } @Test - void shouldFlatMapValuesCapturingErrorsNamed() { - final ValueMapper> mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); - doReturn(List.of("success")).when(mapper).apply("baz"); + void shouldFlatMapValuesWithKeyCapturingErrorsNamed() { + final ValueMapperWithKey> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(List.of("success")).when(mapper).apply("foo", "baz"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -1281,16 +1801,18 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldFlatMapValuesWithKeyCapturingErrors() { + void shouldFlatMapValuesWithKeyCapturingErrorsNamedWithFilter() { final ValueMapperWithKey> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(List.of("success")).when(mapper).apply("foo", "baz"); + doThrow(new RuntimeException("Recoverable")).when(mapper).apply("foo", "qux"); final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapValuesCapturingErrors(mapper); + input.flatMapValuesCapturingErrors(mapper, e -> "Recoverable".equals(e.getMessage()), + Named.as("flatMap")); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1298,7 +1820,8 @@ public void buildTopology(final TopologyBuilder builder) { } }; try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); + final TestInput input = topology.input(); + input.add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); topology.streamOutput("error") @@ -1306,7 +1829,7 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - topology.input().add("foo", "baz"); + input.add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() .hasKey("foo") @@ -1314,20 +1837,32 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); } } @Test - void shouldFlatMapValuesWithKeyCapturingErrorsNamed() { - final ValueMapperWithKey> mapper = mock(); - doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); - doReturn(List.of("success")).when(mapper).apply("foo", "baz"); + void shouldProcessCapturingErrors() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.forward(inputRecord.withKey("success_key").withValue("success_value")); + return; + } + throw new UnsupportedOperationException(); + } + }; final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.flatMapValuesCapturingErrors(mapper, Named.as("flatMap")); + input.processCapturingErrors(processor); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1346,8 +1881,8 @@ public void buildTopology(final TopologyBuilder builder) { topology.input().add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() - .hasKey("foo") - .hasValue("success") + .hasKey("success_key") + .hasValue("success_value") .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); @@ -1355,7 +1890,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldProcessCapturingErrors() { + void shouldProcessCapturingErrorsWithFilter() { final ProcessorSupplier processor = () -> new SimpleProcessor<>() { @Override @@ -1367,6 +1902,9 @@ public void process(final Record inputRecord) { this.forward(inputRecord.withKey("success_key").withValue("success_value")); return; } + if ("foo".equals(inputRecord.key()) && "qux".equals(inputRecord.value())) { + throw new RuntimeException("Recoverable"); + } throw new UnsupportedOperationException(); } }; @@ -1375,7 +1913,7 @@ public void process(final Record inputRecord) { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KErrorStream processed = - input.processCapturingErrors(processor); + input.processCapturingErrors(processor, e -> "Recoverable".equals(e.getMessage())); processed.values().to("output"); processed.errors() .mapValues(ProcessingError::getValue) @@ -1383,7 +1921,8 @@ public void buildTopology(final TopologyBuilder builder) { } }; try (final TestTopology topology = app.startApp()) { - topology.input().add("foo", "bar"); + final TestInput input = topology.input(); + input.add("foo", "bar"); topology.streamOutput("output") .expectNoMoreRecord(); topology.streamOutput("error") @@ -1391,7 +1930,7 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - topology.input().add("foo", "baz"); + input.add("foo", "baz"); topology.streamOutput("output") .expectNextRecord() .hasKey("success_key") @@ -1399,6 +1938,7 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); topology.streamOutput("error") .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); } } @@ -1450,6 +1990,60 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldProcessCapturingErrorsNamedWithFilter() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.forward(inputRecord.withKey("success_key").withValue("success_value")); + return; + } + if ("foo".equals(inputRecord.key()) && "qux".equals(inputRecord.value())) { + throw new RuntimeException("Recoverable"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.processCapturingErrors(processor, e -> "Recoverable".equals(e.getMessage()), + Named.as("process")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + @Test void shouldProcessValuesCapturingErrors() { final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { @@ -1498,6 +2092,59 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldProcessValuesCapturingErrorsWithFilter() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.forward(inputRecord.withValue("success")); + return; + } + if ("foo".equals(inputRecord.key()) && "qux".equals(inputRecord.value())) { + throw new RuntimeException("Recoverable"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.processValuesCapturingErrors(processor, e -> "Recoverable".equals(e.getMessage())); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + @Test void shouldProcessValuesCapturingErrorsNamed() { final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { @@ -1546,6 +2193,60 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldProcessValuesCapturingErrorsNamedWithFilter() { + final FixedKeyProcessorSupplier processor = () -> new SimpleFixedKeyProcessor<>() { + + @Override + public void process(final FixedKeyRecord inputRecord) { + if ("foo".equals(inputRecord.key()) && "bar".equals(inputRecord.value())) { + throw new RuntimeException("Cannot process"); + } + if ("foo".equals(inputRecord.key()) && "baz".equals(inputRecord.value())) { + this.forward(inputRecord.withValue("success")); + return; + } + if ("foo".equals(inputRecord.key()) && "qux".equals(inputRecord.value())) { + throw new RuntimeException("Recoverable"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.processValuesCapturingErrors(processor, e -> "Recoverable".equals(e.getMessage()), + Named.as("process")); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + final TestInput input = topology.input(); + input.add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + input.add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + this.softly.assertThatThrownBy(() -> input.add("foo", "qux")).isInstanceOf(StreamsException.class); + } + } + @Test void shouldRepartition() { final StringApp app = new StringApp() { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index 87d66509..952e2c55 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -453,4 +453,7 @@ public void buildTopology(final TopologyBuilder builder) { } } + //TODO globalTable + //TOOD globalStore + } From d71cf6a66fdefc7161c5742c7ce4cca6c721dfd4 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 14:50:45 +0100 Subject: [PATCH 48/72] Add tests --- .github/workflows/build-and-publish.yaml | 1 + .../main/java/com/bakdata/kafka/KTableX.java | 14 - .../java/com/bakdata/kafka/KTableXImpl.java | 18 - .../java/com/bakdata/kafka/KStreamXTest.java | 5 +- .../java/com/bakdata/kafka/KTableXTest.java | 373 +++++++++++++++++- 5 files changed, 377 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml index 663472f3..5eab07c4 100644 --- a/.github/workflows/build-and-publish.yaml +++ b/.github/workflows/build-and-publish.yaml @@ -12,6 +12,7 @@ jobs: uses: bakdata/ci-templates/.github/workflows/java-gradle-library.yaml@1.62.0 with: java-version: 17 + gradle-refresh-dependencies: true secrets: sonar-token: ${{ secrets.SONARCLOUD_TOKEN }} sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }} diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java index 42f42d26..ac6377d6 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableX.java @@ -337,13 +337,6 @@ KTableX join(KTable other, Function foreignKe KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); - /** - * @see #join(KTable, Function, ValueJoiner, Named, Materialized) - */ - KTableX join(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named, - MaterializedX> materialized); - @Override KTableX join(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, @@ -384,13 +377,6 @@ KTableX leftJoin(KTable other, Function forei KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, Named named, Materialized> materialized); - /** - * @see #leftJoin(KTable, Function, ValueJoiner, Named, Materialized) - */ - KTableX leftJoin(KTable other, Function foreignKeyExtractor, - ValueJoiner joiner, Named named, - MaterializedX> materialized); - @Override KTableX leftJoin(KTable other, Function foreignKeyExtractor, ValueJoiner joiner, TableJoined tableJoined, diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java index baec63ff..73ac9744 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/KTableXImpl.java @@ -463,15 +463,6 @@ public KTableX join(final KTable other, return this.context.wrap(this.wrapped.join(otherTable, foreignKeyExtractor, joiner, named, materialized)); } - @Override - public KTableX join(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named, - final MaterializedX> materialized) { - return this.join(other, foreignKeyExtractor, joiner, named, - materialized.configure(this.context.getConfigurator())); - } - @Override public KTableX join(final KTable other, final Function foreignKeyExtractor, @@ -540,15 +531,6 @@ public KTableX leftJoin(final KTable other, return this.context.wrap(this.wrapped.leftJoin(otherTable, foreignKeyExtractor, joiner, named, materialized)); } - @Override - public KTableX leftJoin(final KTable other, - final Function foreignKeyExtractor, - final ValueJoiner joiner, final Named named, - final MaterializedX> materialized) { - return this.leftJoin(other, foreignKeyExtractor, joiner, named, - materialized.configure(this.context.getConfigurator())); - } - @Override public KTableX leftJoin(final KTable other, final Function foreignKeyExtractor, diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index c36a8033..65575459 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -3611,7 +3611,10 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO errorFilter + //TODO branch + //TODO through + //TODO transform + //TODO process (old) private abstract static class SimpleProcessor implements Processor { private ProcessorContext context = null; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 53ab0483..88a7b7c8 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -28,11 +28,14 @@ import static org.mockito.Mockito.when; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.util.function.Function; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.TableJoined; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.kstream.ValueTransformerWithKey; @@ -1308,7 +1311,375 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO FKey Join + @Test + void shouldFKeyJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = + input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, Named.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinTableJoined() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = + input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, TableJoined.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, Named.as("join"), + Materialized.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFKeyJoinTableJoinedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, TableJoined.as("join"), + Materialized.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoin() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, Function.identity(), + (v1, v2) -> v2 == null ? v1 : v1 + v2); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, Function.identity(), + (v1, v2) -> v2 == null ? v1 : v1 + v2, Named.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinTableJoined() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX otherInput = builder.table("other_input"); + final KTableX joined = input.leftJoin(otherInput, Function.identity(), + (v1, v2) -> v2 == null ? v1 : v1 + v2, TableJoined.as("join")); + joined.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.input("other_input") + .add("bar", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.leftJoin(otherInput, Function.identity(), (v1, v2) -> v2 == null ? v1 : v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.leftJoin(otherInput, Function.identity(), (v1, v2) -> v2 == null ? v1 : v1 + v2, + Named.as("join"), Materialized.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldLeftFKeyJoinTableJoinedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX otherInput = + builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KTableX joined = + input.leftJoin(otherInput, Function.identity(), (v1, v2) -> v2 == null ? v1 : v1 + v2, + TableJoined.as("join"), MaterializedX.with(Serdes.String(), Serdes.String())); + joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.input("other_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("bar", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } private abstract static class SimpleValueTransformer implements ValueTransformerWithKey { From 6355003819b0906f4c6ecafd97cdd88d0833d1e5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 15:11:59 +0100 Subject: [PATCH 49/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 48 +---- .../java/com/bakdata/kafka/KTableXTest.java | 22 --- .../kafka/SimpleFixedKeyProcessor.java | 48 +++++ .../com/bakdata/kafka/SimpleProcessor.java | 48 +++++ .../bakdata/kafka/SimpleValueTransformer.java | 48 +++++ .../bakdata/kafka/TopologyBuilderTest.java | 170 +++++++++++++++++- 6 files changed, 314 insertions(+), 70 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleFixedKeyProcessor.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleProcessor.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 65575459..46dbaa3c 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -53,13 +53,8 @@ import org.apache.kafka.streams.kstream.Printed; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.processor.StateStore; -import org.apache.kafka.streams.processor.api.FixedKeyProcessor; -import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.FixedKeyRecord; -import org.apache.kafka.streams.processor.api.Processor; -import org.apache.kafka.streams.processor.api.ProcessorContext; import org.apache.kafka.streams.processor.api.ProcessorSupplier; import org.apache.kafka.streams.processor.api.Record; import org.apache.kafka.streams.state.KeyValueStore; @@ -236,9 +231,7 @@ public void buildTopology(final TopologyBuilder builder) { input.to((key, value, recordContext) -> key); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() - .outputTopic("output") - .build())) { + try (final TestTopology topology = app.startApp()) { topology.input().add("foo", "bar"); final StreamsConfig streamsConfig = topology.getStreamsConfig(); final Serde keySerde = (Serde) streamsConfig.defaultKeySerde(); @@ -264,9 +257,7 @@ public void buildTopology(final TopologyBuilder builder) { input.to((key, value, recordContext) -> key, ProducedX.with(Serdes.String(), Serdes.String())); } }; - try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() - .outputTopic("output") - .build())) { + try (final TestTopology topology = app.startApp()) { topology.input() .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String()) @@ -3616,39 +3607,4 @@ public void buildTopology(final TopologyBuilder builder) { //TODO transform //TODO process (old) - private abstract static class SimpleProcessor implements Processor { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - protected void forward(final Record outputRecord) { - this.context.forward(outputRecord); - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - - } - - private abstract static class SimpleFixedKeyProcessor implements FixedKeyProcessor { - private FixedKeyProcessorContext context = null; - - @Override - public void init(final FixedKeyProcessorContext context) { - this.context = context; - } - - protected void forward(final FixedKeyRecord outputRecord) { - this.context.forward(outputRecord); - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - - } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 88a7b7c8..977dfc36 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -38,10 +38,7 @@ import org.apache.kafka.streams.kstream.TableJoined; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; -import org.apache.kafka.streams.kstream.ValueTransformerWithKey; import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; -import org.apache.kafka.streams.processor.ProcessorContext; -import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.StoreBuilder; import org.apache.kafka.streams.state.Stores; @@ -1681,23 +1678,4 @@ public void buildTopology(final TopologyBuilder builder) { } } - private abstract static class SimpleValueTransformer - implements ValueTransformerWithKey { - private ProcessorContext context = null; - - @Override - public void init(final ProcessorContext context) { - this.context = context; - } - - @Override - public void close() { - // do nothing - } - - protected S getStateStore(final String name) { - return this.context.getStateStore(name); - } - - } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleFixedKeyProcessor.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleFixedKeyProcessor.java new file mode 100644 index 00000000..fcd4573e --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleFixedKeyProcessor.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.processor.api.FixedKeyProcessor; +import org.apache.kafka.streams.processor.api.FixedKeyProcessorContext; +import org.apache.kafka.streams.processor.api.FixedKeyRecord; + +abstract class SimpleFixedKeyProcessor implements FixedKeyProcessor { + private FixedKeyProcessorContext context = null; + + @Override + public void init(final FixedKeyProcessorContext context) { + this.context = context; + } + + protected void forward(final FixedKeyRecord outputRecord) { + this.context.forward(outputRecord); + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleProcessor.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleProcessor.java new file mode 100644 index 00000000..1b96adab --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleProcessor.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.ProcessorContext; +import org.apache.kafka.streams.processor.api.Record; + +abstract class SimpleProcessor implements Processor { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + protected void forward(final Record outputRecord) { + this.context.forward(outputRecord); + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java new file mode 100644 index 00000000..e7d848a8 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.kstream.ValueTransformerWithKey; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.StateStore; + +abstract class SimpleValueTransformer implements ValueTransformerWithKey { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + @Override + public void close() { + // do nothing + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java index 952e2c55..baec0931 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TopologyBuilderTest.java @@ -29,7 +29,13 @@ import java.util.Map; import java.util.regex.Pattern; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.GlobalKTable; import org.apache.kafka.streams.kstream.Materialized; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; import org.junit.jupiter.api.Test; class TopologyBuilderTest { @@ -453,7 +459,167 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO globalTable - //TOOD globalStore + @Test + void shouldReadGlobalTableFromTopic() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final GlobalKTable otherInput = builder.globalTable("table_input"); + final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadGlobalTableFromTopicUsingConsumed() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final GlobalKTable otherInput = + builder.globalTable("table_input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadGlobalTableFromTopicUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final GlobalKTable otherInput = + builder.globalTable("table_input", MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReadGlobalTableFromTopicUsingConsumedAndMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final GlobalKTable otherInput = + builder.globalTable("table_input", ConsumedX.with(Serdes.String(), Serdes.String()), + Materialized.as("store")); + final KStreamX joined = input.join(otherInput, (k, v) -> k, (v1, v2) -> v1 + v2); + joined.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "baz"); + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldProcessUsingStore() { + final ProcessorSupplier processor = () -> new SimpleProcessor<>() { + + @Override + public void process(final Record inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + final String value = store.get(inputRecord.key()); + this.forward(inputRecord.withValue(inputRecord.value() + value)); + } + }; + final ProcessorSupplier storeUpdater = () -> new SimpleProcessor<>() { + @Override + public void process(final Record inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addGlobalStore(store, "global_input", + ConsumedX.with(Preconfigured.defaultSerde(), Preconfigured.defaultSerde()), storeUpdater); + final KStreamX input = builder.stream("input"); + final KStreamX processed = input.process(processor); + processed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("global_input").add("foo", "baz"); + topology.input("input").add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } } From 144f4744c5c6e31189a46a89f4027cc855a63bfe Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 15:52:35 +0100 Subject: [PATCH 50/72] Add tests --- .../bakdata/kafka/CogroupedStreamXTest.java | 32 ++ .../com/bakdata/kafka/KErrorStreamXTest.java | 112 +++++ .../bakdata/kafka/KGroupedStreamXTest.java | 4 +- .../com/bakdata/kafka/KGroupedTableXTest.java | 387 ++++++++++++++++++ .../java/com/bakdata/kafka/KStreamXTest.java | 4 +- .../java/com/bakdata/kafka/KTableXTest.java | 7 +- .../SessionWindowedCogroupedKStreamXTest.java | 30 ++ .../kafka/SessionWindowedKStreamXTest.java | 32 ++ .../TimeWindowedCogroupedKStreamXTest.java | 30 ++ .../kafka/TimeWindowedKStreamXTest.java | 32 ++ 10 files changed, 664 insertions(+), 6 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java new file mode 100644 index 00000000..4191f6ef --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +class CogroupedStreamXTest { + + //TODO cogroup + //TODO aggregate + //TODO window +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java new file mode 100644 index 00000000..4e1b529a --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java @@ -0,0 +1,112 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.KeyValueMapper; +import org.apache.kafka.streams.kstream.Named; +import org.junit.jupiter.api.Test; + +class KErrorStreamXTest { + + @Test + void shouldGetErrorsAndValues() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGetErrorsAndValuesNamed() { + final KeyValueMapper> mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); + doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapCapturingErrors(mapper); + processed.values(Named.as("values")).to("output"); + processed.errors(Named.as("errors")) + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("success_key") + .hasValue("success_value") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index 583b4d71..5a330549 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -101,7 +101,7 @@ public void buildTopology(final TopologyBuilder builder) { input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); final KTableX counted = grouped.count(MaterializedX.keySerde(Serdes.String())); - counted.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); + counted.toStream().to("output", ProducedX.keySerde(Serdes.String())); } }; try (final TestTopology topology = app.startApp()) { @@ -134,7 +134,7 @@ public void buildTopology(final TopologyBuilder builder) { input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); final KTableX counted = grouped.count(Named.as("count"), MaterializedX.with(Serdes.String(), Serdes.Long())); - counted.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); + counted.toStream().to("output", ProducedX.keySerde(Serdes.String())); } }; try (final TestTopology topology = app.startApp()) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java new file mode 100644 index 00000000..b4edaaa8 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java @@ -0,0 +1,387 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.kstream.Named; +import org.junit.jupiter.api.Test; + +class KGroupedTableXTest { + + @Test + void shouldCount() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k)); + final KTableX counted = grouped.count(); + counted.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k)); + final KTableX counted = grouped.count(Named.as("count")); + counted.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX counted = grouped.count(MaterializedX.keySerde(Serdes.String())); + counted.toStream().to("output", ProducedX.keySerde(Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduce() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k)); + final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, + (value1, value2) -> value1.replace(value2, "")); + reduced.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, + (value1, value2) -> value1.replace(value2, ""), + MaterializedX.with(Serdes.String(), Serdes.String())); + reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceNamedMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, + (value1, value2) -> value1.replace(value2, ""), Named.as("reduce"), + MaterializedX.with(Serdes.String(), Serdes.String())); + reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregate() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k)); + final KTableX reduced = + grouped.aggregate(() -> "", (k, v, a) -> a + v, (k, v, a) -> a.replace(v, "")); + reduced.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k)); + final KTableX reduced = + grouped.aggregate(() -> "", (k, v, a) -> a + v, (k, v, a) -> a.replace(v, ""), + Named.as("aggregate")); + reduced.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX reduced = + grouped.aggregate(() -> "", (k, v, a) -> a + v, (k, v, a) -> a.replace(v, ""), + MaterializedX.with(Serdes.String(), Serdes.String())); + reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX reduced = + grouped.aggregate(() -> "", (k, v, a) -> a + v, (k, v, a) -> a.replace(v, ""), + Named.as("aggregate"), MaterializedX.with(Serdes.String(), Serdes.String())); + reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar") + .add("baz", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("bar") + .hasValue("foobaz") + .expectNextRecord() + .hasKey("bar") + .hasValue("foo") + .expectNextRecord() + .hasKey("qux") + .hasValue("baz") + .expectNoMoreRecord(); + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 46dbaa3c..2c9c8086 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -2849,7 +2849,7 @@ public void buildTopology(final TopologyBuilder builder) { final KGroupedStreamX grouped = input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); final KTableX count = grouped.count(); - count.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); + count.toStream().to("output", ProducedX.keySerde(Serdes.String())); } }; try (final TestTopology topology = app.startApp()) { @@ -2908,7 +2908,7 @@ public void buildTopology(final TopologyBuilder builder) { final KGroupedStreamX grouped = input.groupBy((k, v) -> v, GroupedX.with(Serdes.String(), Serdes.String())); final KTableX count = grouped.count(); - count.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.Long())); + count.toStream().to("output", ProducedX.keySerde(Serdes.String())); } }; try (final TestTopology topology = app.startApp()) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index 977dfc36..d9567b26 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -423,7 +423,7 @@ public void buildTopology(final TopologyBuilder builder) { final KGroupedTableX grouped = input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); grouped.count().toStream() - .to("output", ProducedX.with(Serdes.String(), Serdes.Long())); + .to("output", ProducedX.keySerde(Serdes.String())); } }; try (final TestTopology topology = app.startApp()) { @@ -1460,7 +1460,7 @@ public void buildTopology(final TopologyBuilder builder) { builder.table("other_input", ConsumedX.with(Serdes.String(), Serdes.String())); final KTableX joined = input.join(otherInput, Function.identity(), (v1, v2) -> v1 + v2, TableJoined.as("join"), - Materialized.with(Serdes.String(), Serdes.String())); + MaterializedX.with(Serdes.String(), Serdes.String())); joined.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); } }; @@ -1678,4 +1678,7 @@ public void buildTopology(final TopologyBuilder builder) { } } + //TODO suppress + //TODO queryable store name + } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java new file mode 100644 index 00000000..793bc1dc --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java @@ -0,0 +1,30 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +class SessionWindowedCogroupedKStreamXTest { + + //TODO aggregate +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java new file mode 100644 index 00000000..8e18f281 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +class SessionWindowedKStreamXTest { + + //TODO count + //TODO aggregate + //TODO reduce +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java new file mode 100644 index 00000000..ffb5f786 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java @@ -0,0 +1,30 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +class TimeWindowedCogroupedKStreamXTest { + + //TODO aggregate +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java new file mode 100644 index 00000000..b1816939 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java @@ -0,0 +1,32 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +class TimeWindowedKStreamXTest { + + //TODO count + //TODO aggregate + //TODO reduce +} From 34a96344f1aa81257a0139f8b1f28e2252ea8a1e Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 17:05:39 +0100 Subject: [PATCH 51/72] Add tests --- .../com/bakdata/kafka/KErrorStreamXTest.java | 79 ++- .../com/bakdata/kafka/KGroupedTableXTest.java | 33 ++ .../kafka/TimeWindowedKStreamXTest.java | 490 +++++++++++++++++- 3 files changed, 597 insertions(+), 5 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java index 4e1b529a..7560a232 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KErrorStreamXTest.java @@ -32,12 +32,13 @@ import org.apache.kafka.streams.KeyValue; import org.apache.kafka.streams.kstream.KeyValueMapper; import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.ValueMapper; import org.junit.jupiter.api.Test; class KErrorStreamXTest { @Test - void shouldGetErrorsAndValues() { + void shouldGetErrorsAndValuesWithKey() { final KeyValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); @@ -74,7 +75,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldGetErrorsAndValuesNamed() { + void shouldGetErrorsAndValuesWithKeyNamed() { final KeyValueMapper> mapper = mock(); doThrow(new RuntimeException("Cannot process")).when(mapper).apply("foo", "bar"); doReturn(KeyValue.pair("success_key", "success_value")).when(mapper).apply("foo", "baz"); @@ -109,4 +110,78 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); } } + + @Test + void shouldGetErrorsAndValues() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values().to("output"); + processed.errors() + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } + + @Test + void shouldGetErrorsAndValuesNamed() { + final ValueMapper mapper = mock(); + doThrow(new RuntimeException("Cannot process")).when(mapper).apply("bar"); + doReturn("success").when(mapper).apply("baz"); + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KErrorStream processed = + input.mapValuesCapturingErrors(mapper); + processed.values(Named.as("value")).to("output"); + processed.errors(Named.as("errors")) + .mapValues(ProcessingError::getValue) + .to("error"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput("output") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input().add("foo", "baz"); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("success") + .expectNoMoreRecord(); + topology.streamOutput("error") + .expectNoMoreRecord(); + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java index b4edaaa8..4089be36 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedTableXTest.java @@ -118,6 +118,39 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldCountNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = + builder.table("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedTableX grouped = + input.groupBy((k, v) -> KeyValue.pair(v, k), GroupedX.with(Serdes.String(), Serdes.String())); + final KTableX counted = + grouped.count(Named.as("count"), MaterializedX.keySerde(Serdes.String())); + counted.toStream().to("output", ProducedX.keySerde(Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("baz", "bar"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("bar") + .hasValue(1L) + .expectNextRecord() + .hasKey("bar") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + @Test void shouldReduce() { final StringApp app = new StringApp() { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java index b1816939..6b9dc9b4 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java @@ -24,9 +24,493 @@ package com.bakdata.kafka; +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.junit.jupiter.api.Test; + class TimeWindowedKStreamXTest { - //TODO count - //TODO aggregate - //TODO reduce + @Test + void shouldCount() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, Long> counted = windowed.count(); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:60000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, Long> counted = windowed.count(Named.as("count")); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:60000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, Long> counted = + windowed.count(MaterializedX.with(Serdes.String(), Serdes.Long())); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output", ProducedX.keySerde(Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:60000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, Long> counted = + windowed.count(MaterializedX.with(Serdes.String(), Serdes.Long())); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output", ProducedX.keySerde(Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:60000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregate() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = windowed.aggregate(() -> "", (k, v, a) -> a + v); + final KStreamX output = + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v, a) -> a + v, Named.as("aggregate")); + final KStreamX output = + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = windowed.aggregate(() -> "", (k, v, a) -> a + v, + MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v, a) -> a + v, Named.as("aggregate"), + MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduce() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> reduced = windowed.reduce((v1, v2) -> v1 + v2); + final KStreamX output = + reduced.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> reduced = + windowed.reduce((v1, v2) -> v1 + v2, Named.as("reduce")); + final KStreamX output = + reduced.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> reduced = + windowed.reduce((v1, v2) -> v1 + v2, MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = + reduced.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> reduced = + windowed.reduce((v1, v2) -> v1 + v2, Named.as("reduce"), + MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = + reduced.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } } From e0bd7d96145ba830f6305d0d8934cc1bf1daee7a Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 18:57:04 +0100 Subject: [PATCH 52/72] Add tests --- .../kafka/SessionWindowedKStreamXTest.java | 566 +++++++++++++++++- .../kafka/TimeWindowedKStreamXTest.java | 4 +- 2 files changed, 566 insertions(+), 4 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java index 8e18f281..a370c528 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java @@ -24,9 +24,569 @@ package com.bakdata.kafka; +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.junit.jupiter.api.Test; + class SessionWindowedKStreamXTest { - //TODO count - //TODO aggregate - //TODO reduce + @Test + void shouldCount() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = windowed.count(); + final KStreamX output = counted.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = windowed.count(Named.as("count")); + final KStreamX output = counted.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = + windowed.count(MaterializedX.with(Serdes.String(), Serdes.Long())); + final KStreamX output = counted.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", ProducedX.keySerde(Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldCountNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = + windowed.count(Named.as("count"), MaterializedX.with(Serdes.String(), Serdes.Long())); + final KStreamX output = counted.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", ProducedX.keySerde(Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(1L) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue(2L) + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregate() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v, a) -> a + v, (k, a1, a2) -> a1 + a2); + final KStreamX output = aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v, a) -> a + v, (k, a1, a2) -> a1 + a2, Named.as("aggregate")); + final KStreamX output = aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v, a) -> a + v, (k, a1, a2) -> a1 + a2, + MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v, a) -> a + v, (k, a1, a2) -> a1 + a2, Named.as("aggregate"), + MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduce() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> reduced = windowed.reduce((v1, v2) -> v1 + v2); + final KStreamX output = reduced.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> reduced = + windowed.reduce((v1, v2) -> v1 + v2, Named.as("reduce")); + final KStreamX output = reduced.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> reduced = + windowed.reduce((v1, v2) -> v1 + v2, MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = reduced.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldReduceNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> reduced = + windowed.reduce((v1, v2) -> v1 + v2, Named.as("reduce"), + MaterializedX.with(Serdes.String(), Serdes.String())); + final KStreamX output = reduced.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + //TODO emitStrategy } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java index 6b9dc9b4..fd9086b2 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java @@ -163,7 +163,7 @@ public void buildTopology(final TopologyBuilder builder) { final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, Long> counted = - windowed.count(MaterializedX.with(Serdes.String(), Serdes.Long())); + windowed.count(Named.as("count"), MaterializedX.with(Serdes.String(), Serdes.Long())); final KStreamX output = counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); output.to("output", ProducedX.keySerde(Serdes.String())); @@ -513,4 +513,6 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); } } + + //TODO emitStrategy } From 9ddefc841448fb4d69cfcbcbbe65e961f377a256 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 19:17:20 +0100 Subject: [PATCH 53/72] Add tests --- .../bakdata/kafka/CogroupedStreamXTest.java | 309 +++++++++++++++++- 1 file changed, 306 insertions(+), 3 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java index 4191f6ef..3a2d0a2f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java @@ -24,9 +24,312 @@ package com.bakdata.kafka; +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.SlidingWindows; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.junit.jupiter.api.Test; + class CogroupedStreamXTest { - //TODO cogroup - //TODO aggregate - //TODO window + @Test + void shouldAggregate() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final KTableX aggregated = cogrouped.aggregate(() -> ""); + aggregated.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final KTableX aggregated = cogrouped.aggregate(() -> "", Named.as("cogroup")); + aggregated.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final KTableX aggregated = + cogrouped.aggregate(() -> "", MaterializedX.with(Serdes.String(), Serdes.String())); + aggregated.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final KTableX aggregated = cogrouped.aggregate(() -> "", Named.as("cogroup"), + MaterializedX.with(Serdes.String(), Serdes.String())); + aggregated.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .add("foo", "baz"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldCogroup() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final KStreamX otherInput = builder.stream("other_input"); + final KGroupedStreamX otherGrouped = otherInput.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final CogroupedKStreamX + cocogrouped = cogrouped.cogroup(otherGrouped, (k, v1, v2) -> v2 + v1); + final KTableX aggregated = cocogrouped.aggregate(() -> ""); + aggregated.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.input("other_input") + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barqux") + .expectNoMoreRecord(); + topology.input("input") + .add("foo", "baz"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barquxbaz") + .expectNoMoreRecord(); + topology.input("other_input") + .add("foo", "quux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barquxbazquux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTimeWindow() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final TimeWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = windowed.aggregate(() -> ""); + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldSlidingWindow() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final TimeWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(SlidingWindows.ofTimeDifferenceWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = windowed.aggregate(() -> ""); + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:1") + .hasValue("baz") + .expectNextRecord() + .hasKey("foo:30001") + .hasValue("qux") + .expectNextRecord() + .hasKey("foo:30000") + .hasValue("bazqux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldSessionWindow() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final SessionWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v1, v2) -> v1 + v2); + aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } } From 24b2b36ff1acec950e65932374991fc592f6b1fb Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 19:46:16 +0100 Subject: [PATCH 54/72] Add tests --- .../SessionWindowedCogroupedKStreamXTest.java | 200 +++++++++++++++- .../com/bakdata/kafka/StreamsContextTest.java | 218 ++++++++++++++++++ .../TimeWindowedCogroupedKStreamXTest.java | 169 +++++++++++++- 3 files changed, 585 insertions(+), 2 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsContextTest.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java index 793bc1dc..5c0ecbda 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java @@ -24,7 +24,205 @@ package com.bakdata.kafka; +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.junit.jupiter.api.Test; + class SessionWindowedCogroupedKStreamXTest { - //TODO aggregate + @Test + void shouldAggregate() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final SessionWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v1, v2) -> v1 + v2); + aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final SessionWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v1, v2) -> v1 + v2, Named.as("aggregate")); + aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final SessionWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v1, v2) -> v1 + v2, + MaterializedX.with(Serdes.String(), Serdes.String())); + aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()) + .to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final SessionWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", (k, v1, v2) -> v1 + v2, Named.as("aggregate"), + MaterializedX.with(Serdes.String(), Serdes.String())); + aggregated.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()) + .to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:30000:30000") + .hasValue(null) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:91000:91000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsContextTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsContextTest.java new file mode 100644 index 00000000..5a6c9e61 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamsContextTest.java @@ -0,0 +1,218 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static java.util.Collections.emptyMap; + +import java.time.Duration; +import org.apache.kafka.streams.KeyValue; +import org.apache.kafka.streams.StreamsBuilder; +import org.apache.kafka.streams.kstream.BranchedKStream; +import org.apache.kafka.streams.kstream.CogroupedKStream; +import org.apache.kafka.streams.kstream.KGroupedStream; +import org.apache.kafka.streams.kstream.KGroupedTable; +import org.apache.kafka.streams.kstream.KStream; +import org.apache.kafka.streams.kstream.KTable; +import org.apache.kafka.streams.kstream.SessionWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.SessionWindowedKStream; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.TimeWindowedCogroupedKStream; +import org.apache.kafka.streams.kstream.TimeWindowedKStream; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class StreamsContextTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + private static StreamsContext newContext() { + return new StreamsContext(StreamsTopicConfig.builder().build(), new Configurator(emptyMap())); + } + + @Test + void shouldNotReWrapStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KStreamX wrapped = context.wrap(stream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapGroupedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final KGroupedStreamX wrapped = context.wrap(groupedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapTimeWindowedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final TimeWindowedKStream windowedStream = + groupedStream.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(1L))); + final TimeWindowedKStreamX wrapped = context.wrap(windowedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapSessionWindowedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final SessionWindowedKStream windowedStream = + groupedStream.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(1L))); + final SessionWindowedKStreamX wrapped = context.wrap(windowedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapBranchedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final BranchedKStream branchedStream = stream.split(); + final BranchedKStreamX wrapped = context.wrap(branchedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapCogroupedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final CogroupedKStream cogroupedStream = groupedStream.cogroup((k, v, a) -> a + v); + final CogroupedKStreamX wrapped = context.wrap(cogroupedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapTimeWindowedCogroupedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final CogroupedKStream cogroupedStream = groupedStream.cogroup((k, v, a) -> a + v); + final TimeWindowedCogroupedKStream windowedCogroupedStream = + cogroupedStream.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(1L))); + final TimeWindowedCogroupedKStreamX wrapped = context.wrap(windowedCogroupedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapSessionWindowedCogroupedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final CogroupedKStream cogroupedStream = groupedStream.cogroup((k, v, a) -> a + v); + final SessionWindowedCogroupedKStream windowedCogroupedStream = + cogroupedStream.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(1L))); + final SessionWindowedCogroupedKStreamX wrapped = context.wrap(windowedCogroupedStream); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapTable() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KTable table = builder.table("topic"); + final KTableX wrapped = context.wrap(table); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldNotReWrapGroupedTable() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KTable table = builder.table("topic"); + final KGroupedTable groupedTable = table.groupBy((k, v) -> KeyValue.pair(v, k)); + final KGroupedTableX wrapped = context.wrap(groupedTable); + this.softly.assertThat(context.wrap(wrapped)).isSameAs(wrapped); + } + + @Test + void shouldUnwrapStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KStreamX wrapped = context.wrap(stream); + this.softly.assertThat(StreamsContext.maybeUnwrap(wrapped)).isSameAs(stream); + } + + @Test + void shouldNotUnwrapStream() { + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + this.softly.assertThat(StreamsContext.maybeUnwrap(stream)).isSameAs(stream); + } + + @Test + void shouldUnwrapGroupedStream() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + final KGroupedStreamX wrapped = context.wrap(groupedStream); + this.softly.assertThat(StreamsContext.maybeUnwrap(wrapped)).isSameAs(groupedStream); + } + + @Test + void shouldNotUnwrapGroupedStream() { + final StreamsBuilder builder = new StreamsBuilder(); + final KStream stream = builder.stream("topic"); + final KGroupedStream groupedStream = stream.groupByKey(); + this.softly.assertThat(StreamsContext.maybeUnwrap(groupedStream)).isSameAs(groupedStream); + } + + @Test + void shouldUnwrapTable() { + final StreamsContext context = newContext(); + final StreamsBuilder builder = new StreamsBuilder(); + final KTable table = builder.table("topic"); + final KTableX wrapped = context.wrap(table); + this.softly.assertThat(StreamsContext.maybeUnwrap(wrapped)).isSameAs(table); + } + + @Test + void shouldNotUnwrapTable() { + final StreamsBuilder builder = new StreamsBuilder(); + final KTable table = builder.table("topic"); + this.softly.assertThat(StreamsContext.maybeUnwrap(table)).isSameAs(table); + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java index ffb5f786..a03597f7 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java @@ -24,7 +24,174 @@ package com.bakdata.kafka; +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.Named; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.junit.jupiter.api.Test; + class TimeWindowedCogroupedKStreamXTest { - //TODO aggregate + @Test + void shouldAggregate() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final TimeWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = windowed.aggregate(() -> ""); + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final TimeWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", Named.as("aggregate")); + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final TimeWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = + windowed.aggregate(() -> "", MaterializedX.with(Serdes.String(), Serdes.String())); + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()) + .to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldAggregateNamedUsingMaterialized() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = + input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final CogroupedKStreamX cogrouped = + grouped.cogroup((key, value, aggregate) -> aggregate + value); + final TimeWindowedCogroupedKStreamX windowed = + cogrouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, String> aggregated = windowed.aggregate(() -> "", Named.as("aggregate"), + MaterializedX.with(Serdes.String(), Serdes.String())); + aggregated.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()) + .to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "qux"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue("bar") + .expectNextRecord() + .hasKey("foo:0") + .hasValue("barbaz") + .expectNextRecord() + .hasKey("foo:60000") + .hasValue("qux") + .expectNoMoreRecord(); + } + } } From 8a7ab88ef2246ad5e7425f08af3867c61462c556 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 21:37:37 +0100 Subject: [PATCH 55/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 105 +++++++++++++++++- .../java/com/bakdata/kafka/KTableXTest.java | 37 +++++- .../kafka/SessionWindowedKStreamXTest.java | 36 +++++- .../kafka/TimeWindowedKStreamXTest.java | 33 +++++- 4 files changed, 205 insertions(+), 6 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 2c9c8086..1c41883f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -51,6 +51,7 @@ import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Printed; +import org.apache.kafka.streams.kstream.Produced; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; @@ -3108,6 +3109,48 @@ public void buildTopology(final TopologyBuilder builder) { } } + @Test + void shouldBranch() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX[] branches = input.branch((k, v) -> true); + branches[0].to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldBranchNamed() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX[] branches = input.branch(Named.as("branch"), (k, v) -> true); + branches[0].to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + @Test void shouldTableJoin() { final StringApp app = new StringApp() { @@ -3602,8 +3645,66 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO branch - //TODO through + @Test + void shouldRouteThrough() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX through = input.through("intermediate"); + through.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput("intermediate") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.streamOutput("output") + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldRouteThroughUsingProduced() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + final KStreamX through = + input.through("intermediate", Produced.with(Serdes.String(), Serdes.String())); + through.to("output", ProducedX.with(Serdes.String(), Serdes.String())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + topology.streamOutput("intermediate") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + topology.streamOutput("output") + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + //TODO transform //TODO process (old) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index d9567b26..e2289afa 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -24,10 +24,12 @@ package com.bakdata.kafka; +import static java.util.Collections.emptyMap; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; import java.util.function.Function; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; @@ -35,6 +37,8 @@ import org.apache.kafka.streams.kstream.Materialized; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.Predicate; +import org.apache.kafka.streams.kstream.Suppressed; +import org.apache.kafka.streams.kstream.Suppressed.BufferConfig; import org.apache.kafka.streams.kstream.TableJoined; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; @@ -1678,7 +1682,36 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO suppress - //TODO queryable store name + @Test + void shouldSuppress() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX input = builder.table("input"); + final KTableX suppressed = + input.suppress(Suppressed.untilTimeLimit(Duration.ofSeconds(60L), BufferConfig.unbounded())); + suppressed.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .add("foo", "baz") + .at(60000L) + .add("baz", "qux"); // trigger flush + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldHaveQueryableStoreName() { + final TopologyBuilder builder = new TopologyBuilder(StreamsTopicConfig.builder().build(), emptyMap()); + final KTableX table = builder.stream("input").toTable(Materialized.as("store")); + this.softly.assertThat(table.queryableStoreName()).isEqualTo("store"); + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java index a370c528..e07f4ff5 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java @@ -27,6 +27,7 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.time.Duration; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.EmitStrategy; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.SessionWindows; import org.apache.kafka.streams.kstream.Windowed; @@ -588,5 +589,38 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO emitStrategy + @Test + void shouldUseEmitStrategy() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); + final KTableX, Long> counted = + windowed.emitStrategy(EmitStrategy.onWindowClose()).count(); + final KStreamX output = counted.toStream( + (k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli() + ":" + k.window().endTime() + .toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + // if time is lower than session size, there is a bug in Kafka Streams + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "bar") + .at(Duration.ofSeconds(60L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(91L).toMillis()) // trigger flush + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:30000:60000") + .hasValue(2L) + .expectNoMoreRecord(); + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java index fd9086b2..7a88d106 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java @@ -27,6 +27,7 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import java.time.Duration; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.EmitStrategy; import org.apache.kafka.streams.kstream.Named; import org.apache.kafka.streams.kstream.TimeWindows; import org.apache.kafka.streams.kstream.Windowed; @@ -514,5 +515,35 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO emitStrategy + @Test + void shouldUseEmitStrategy() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); + final KTableX, Long> counted = + windowed.emitStrategy(EmitStrategy.onWindowClose()).count(); + final KStreamX output = + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()); + output.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar") + .at(Duration.ofSeconds(30L).toMillis()) + .add("foo", "baz") + .at(Duration.ofSeconds(60L).toMillis()) // trigger flush + .add("foo", "qux"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(2L) + .expectNoMoreRecord(); + } + } } From 62adf8b4480720dac7b0651c96b5f0204ea3f6fe Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 22:05:03 +0100 Subject: [PATCH 56/72] Add tests --- .../java/com/bakdata/kafka/StoresXTest.java | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java new file mode 100644 index 00000000..adcc0a14 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java @@ -0,0 +1,315 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import java.util.Set; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.processor.api.Processor; +import org.apache.kafka.streams.processor.api.ProcessorSupplier; +import org.apache.kafka.streams.processor.api.Record; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.SessionStore; +import org.apache.kafka.streams.state.StoreBuilder; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.TimestampedKeyValueStore; +import org.apache.kafka.streams.state.TimestampedWindowStore; +import org.apache.kafka.streams.state.ValueAndTimestamp; +import org.apache.kafka.streams.state.VersionedKeyValueStore; +import org.apache.kafka.streams.state.WindowStore; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class StoresXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldCreateKeyValueStore() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.process(new ProcessorSupplier<>() { + @Override + public Processor get() { + return new SimpleProcessor<>() { + @Override + public void process(final Record inputRecord) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + } + + @Override + public Set> stores() { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Serdes.String(), + Serdes.String()); + return Set.of(store); + } + }); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldCreateSessionStore() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX, String> input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())) + .groupByKey() + .windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))) + .reduce((v1, v2) -> v1 + v2); + input.toStream().process(new ProcessorSupplier<>() { + @Override + public Processor, String, String, String> get() { + return new SimpleProcessor<>() { + @Override + public void process(final Record, String> inputRecord) { + final SessionStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value()); + } + }; + } + + @Override + public Set> stores() { + final StoreBuilder> store = builder.stores() + .sessionStoreBuilder(Stores.inMemorySessionStore("my-store", Duration.ofSeconds(60L)), + Serdes.String(), + Serdes.String()); + return Set.of(store); + } + }); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final SessionStore store = + topology.getTestDriver().getSessionStore("my-store"); + this.softly.assertThat(store.fetchSession("foo", 0, 60000)).isEqualTo("bar"); + } + } + + @Test + void shouldCreateTimestampedWindowStore() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.process(new ProcessorSupplier<>() { + @Override + public Processor get() { + return new SimpleProcessor<>() { + @Override + public void process(final Record inputRecord) { + final TimestampedWindowStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), + ValueAndTimestamp.make(inputRecord.value(), inputRecord.timestamp()), + inputRecord.timestamp()); + } + }; + } + + @Override + public Set> stores() { + final StoreBuilder> store = builder.stores() + .timestampedWindowStoreBuilder( + Stores.inMemoryWindowStore("my-store", Duration.ofSeconds(60L), + Duration.ofSeconds(60L), false), + Serdes.String(), + Serdes.String()); + return Set.of(store); + } + }); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final WindowStore> store = + topology.getTestDriver().getTimestampedWindowStore("my-store"); + this.softly.assertThat(store.fetch("foo", 0L)).isEqualTo(ValueAndTimestamp.make("bar", 0L)); + } + } + + @Test + void shouldCreateWindowStore() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.process(new ProcessorSupplier<>() { + @Override + public Processor get() { + return new SimpleProcessor<>() { + @Override + public void process(final Record inputRecord) { + final WindowStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value(), inputRecord.timestamp()); + } + }; + } + + @Override + public Set> stores() { + final StoreBuilder> store = builder.stores() + .windowStoreBuilder(Stores.inMemoryWindowStore("my-store", Duration.ofSeconds(60L), + Duration.ofSeconds(60L), false), Serdes.String(), + Serdes.String()); + return Set.of(store); + } + }); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final WindowStore store = + topology.getTestDriver().getWindowStore("my-store"); + this.softly.assertThat(store.fetch("foo", 0L)).isEqualTo("bar"); + } + } + + @Test + void shouldCreateVersionedKeyValueStore() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.process(new ProcessorSupplier<>() { + @Override + public Processor get() { + return new SimpleProcessor<>() { + @Override + public void process(final Record inputRecord) { + final VersionedKeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), inputRecord.value(), inputRecord.timestamp()); + } + }; + } + + @Override + public Set> stores() { + final StoreBuilder> store = builder.stores() + .versionedKeyValueStoreBuilder( + Stores.persistentVersionedKeyValueStore("my-store", Duration.ofSeconds(60L)), + Serdes.String(), + Serdes.String()); + return Set.of(store); + } + }); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final VersionedKeyValueStore store = + topology.getTestDriver().getVersionedKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo", 0L)) + .satisfies(versionedRecord -> { + this.softly.assertThat(versionedRecord.value()).isEqualTo("bar"); + this.softly.assertThat(versionedRecord.timestamp()).isEqualTo(0L); + this.softly.assertThat(versionedRecord.validTo()).isNotPresent(); + }); + } + } + + @Test + void shouldCreateTimestampedKeyValueStore() { + final DoubleApp app = new DoubleApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); + input.process(new ProcessorSupplier<>() { + @Override + public Processor get() { + return new SimpleProcessor<>() { + @Override + public void process(final Record inputRecord) { + final TimestampedKeyValueStore store = this.getStateStore("my-store"); + store.put(inputRecord.key(), + ValueAndTimestamp.make(inputRecord.value(), inputRecord.timestamp())); + } + }; + } + + @Override + public Set> stores() { + final StoreBuilder> store = builder.stores() + .timestampedKeyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), + Serdes.String(), + Serdes.String()); + return Set.of(store); + } + }); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .add("foo", "bar"); + final KeyValueStore> store = + topology.getTestDriver().getTimestampedKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo(ValueAndTimestamp.make("bar", 0L)); + } + } +} From d66d25032e43dbed9d9a106c9a637369e6c7ca51 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Mon, 17 Feb 2025 22:16:55 +0100 Subject: [PATCH 57/72] Add tests --- streams-bootstrap-core/build.gradle.kts | 2 +- .../bakdata/kafka/CogroupedStreamXTest.java | 6 ++---- .../com/bakdata/kafka/KGroupedStreamXTest.java | 18 ++++++------------ .../java/com/bakdata/kafka/KStreamXTest.java | 10 +++++----- .../SessionWindowedCogroupedKStreamXTest.java | 6 ++---- .../kafka/SessionWindowedKStreamXTest.java | 18 ++++++------------ .../java/com/bakdata/kafka/StoresXTest.java | 8 ++------ .../TimeWindowedCogroupedKStreamXTest.java | 6 ++---- .../kafka/TimeWindowedKStreamXTest.java | 18 ++++++------------ 9 files changed, 32 insertions(+), 60 deletions(-) diff --git a/streams-bootstrap-core/build.gradle.kts b/streams-bootstrap-core/build.gradle.kts index edb26217..be3abf1e 100644 --- a/streams-bootstrap-core/build.gradle.kts +++ b/streams-bootstrap-core/build.gradle.kts @@ -20,7 +20,7 @@ dependencies { ) implementation(group = "org.jooq", name = "jool", version = "0.9.15") implementation(group = "io.github.resilience4j", name = "resilience4j-retry", version = "1.7.1") - api(group = "com.bakdata.kafka", name = "error-handling-core", version = "1.6.1-SNAPSHOT") + api(group = "com.bakdata.kafka", name = "error-handling-core", version = "1.6.1") val junitVersion: String by project testRuntimeOnly(group = "org.junit.jupiter", name = "junit-jupiter-engine", version = junitVersion) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java index 3a2d0a2f..ff09177a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/CogroupedStreamXTest.java @@ -99,8 +99,7 @@ void shouldAggregateUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final CogroupedKStreamX cogrouped = grouped.cogroup((key, value, aggregate) -> aggregate + value); final KTableX aggregated = @@ -134,8 +133,7 @@ void shouldAggregateNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final CogroupedKStreamX cogrouped = grouped.cogroup((key, value, aggregate) -> aggregate + value); final KTableX aggregated = cogrouped.aggregate(() -> "", Named.as("cogroup"), diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java index 5a330549..5dbf19b3 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KGroupedStreamXTest.java @@ -97,8 +97,7 @@ void shouldCountUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final KTableX counted = grouped.count(MaterializedX.keySerde(Serdes.String())); counted.toStream().to("output", ProducedX.keySerde(Serdes.String())); @@ -130,8 +129,7 @@ void shouldCountNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final KTableX counted = grouped.count(Named.as("count"), MaterializedX.with(Serdes.String(), Serdes.Long())); counted.toStream().to("output", ProducedX.keySerde(Serdes.String())); @@ -189,8 +187,7 @@ void shouldReduceUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, MaterializedX.with(Serdes.String(), Serdes.String())); reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); @@ -222,8 +219,7 @@ void shouldReduceNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final KTableX reduced = grouped.reduce((value1, value2) -> value1 + value2, Named.as("reduce"), MaterializedX.with(Serdes.String(), Serdes.String())); reduced.toStream().to("output", ProducedX.with(Serdes.String(), Serdes.String())); @@ -282,8 +278,7 @@ void shouldAggregateUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final KTableX aggregated = grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value, MaterializedX.with(Serdes.String(), Serdes.String())); @@ -317,8 +312,7 @@ void shouldAggregateNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final KTableX aggregated = grouped.aggregate(() -> "", (key, value, aggregate) -> aggregate + value, Named.as("aggregate"), MaterializedX.with(Serdes.String(), Serdes.String())); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 1c41883f..aba48664 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -2847,8 +2847,8 @@ void shouldGroupByKeyUsingGrouped() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.selectKey((k, v) -> v) // repartition + .groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); final KTableX count = grouped.count(); count.toStream().to("output", ProducedX.keySerde(Serdes.String())); } @@ -2858,15 +2858,15 @@ public void buildTopology(final TopologyBuilder builder) { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String()) .add("foo", "bar") - .add("foo", "baz"); + .add("baz", "bar"); topology.streamOutput() .withKeySerde(Serdes.String()) .withValueSerde(Serdes.Long()) .expectNextRecord() - .hasKey("foo") + .hasKey("bar") .hasValue(1L) .expectNextRecord() - .hasKey("foo") + .hasKey("bar") .hasValue(2L) .expectNoMoreRecord(); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java index 5c0ecbda..4102ecee 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedCogroupedKStreamXTest.java @@ -129,8 +129,7 @@ void shouldAggregateUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final CogroupedKStreamX cogrouped = grouped.cogroup((key, value, aggregate) -> aggregate + value); final SessionWindowedCogroupedKStreamX windowed = @@ -181,8 +180,7 @@ void shouldAggregateNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final CogroupedKStreamX cogrouped = grouped.cogroup((key, value, aggregate) -> aggregate + value); final SessionWindowedCogroupedKStreamX windowed = diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java index e07f4ff5..018da7e6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SessionWindowedKStreamXTest.java @@ -128,8 +128,7 @@ void shouldCountUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final SessionWindowedKStreamX windowed = grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); final KTableX, Long> counted = @@ -177,8 +176,7 @@ void shouldCountNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final SessionWindowedKStreamX windowed = grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); final KTableX, Long> counted = @@ -312,8 +310,7 @@ void shouldAggregateUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final SessionWindowedKStreamX windowed = grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); final KTableX, String> aggregated = @@ -362,8 +359,7 @@ void shouldAggregateNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final SessionWindowedKStreamX windowed = grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); final KTableX, String> aggregated = @@ -497,8 +493,7 @@ void shouldReduceUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final SessionWindowedKStreamX windowed = grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); final KTableX, String> reduced = @@ -546,8 +541,7 @@ void shouldReduceNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final SessionWindowedKStreamX windowed = grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofSeconds(30L))); final KTableX, String> reduced = diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java index adcc0a14..441cef75 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StoresXTest.java @@ -41,6 +41,7 @@ import org.apache.kafka.streams.state.TimestampedWindowStore; import org.apache.kafka.streams.state.ValueAndTimestamp; import org.apache.kafka.streams.state.VersionedKeyValueStore; +import org.apache.kafka.streams.state.VersionedRecord; import org.apache.kafka.streams.state.WindowStore; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; @@ -262,12 +263,7 @@ public Set> stores() { .add("foo", "bar"); final VersionedKeyValueStore store = topology.getTestDriver().getVersionedKeyValueStore("my-store"); - this.softly.assertThat(store.get("foo", 0L)) - .satisfies(versionedRecord -> { - this.softly.assertThat(versionedRecord.value()).isEqualTo("bar"); - this.softly.assertThat(versionedRecord.timestamp()).isEqualTo(0L); - this.softly.assertThat(versionedRecord.validTo()).isNotPresent(); - }); + this.softly.assertThat(store.get("foo", 0L)).isEqualTo(new VersionedRecord<>("bar", 0L)); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java index a03597f7..4aaf613a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedCogroupedKStreamXTest.java @@ -114,8 +114,7 @@ void shouldAggregateUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final CogroupedKStreamX cogrouped = grouped.cogroup((key, value, aggregate) -> aggregate + value); final TimeWindowedCogroupedKStreamX windowed = @@ -158,8 +157,7 @@ void shouldAggregateNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final CogroupedKStreamX cogrouped = grouped.cogroup((key, value, aggregate) -> aggregate + value); final TimeWindowedCogroupedKStreamX windowed = diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java index 7a88d106..17d34a59 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TimeWindowedKStreamXTest.java @@ -116,8 +116,7 @@ void shouldCountUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, Long> counted = @@ -159,8 +158,7 @@ void shouldCountNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, Long> counted = @@ -275,8 +273,7 @@ void shouldAggregateUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, String> aggregated = windowed.aggregate(() -> "", (k, v, a) -> a + v, @@ -318,8 +315,7 @@ void shouldAggregateNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, String> aggregated = @@ -435,8 +431,7 @@ void shouldReduceUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, String> reduced = @@ -478,8 +473,7 @@ void shouldReduceNamedUsingMaterialized() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.with(Serdes.String(), Serdes.String())); - final KGroupedStreamX grouped = - input.groupByKey(GroupedX.with(Serdes.String(), Serdes.String())); + final KGroupedStreamX grouped = input.groupByKey(); final TimeWindowedKStreamX windowed = grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofSeconds(60L))); final KTableX, String> reduced = From 1251b543e2edc3cfa3300e77b934a63441e51cbd Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 09:55:35 +0100 Subject: [PATCH 58/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 744 +++++++++++++++++- .../java/com/bakdata/kafka/KTableXTest.java | 16 +- .../com/bakdata/kafka/SimpleTransformer.java | 48 ++ .../bakdata/kafka/SimpleValueTransformer.java | 4 +- .../kafka/SimpleValueTransformerWithKey.java | 48 ++ 5 files changed, 849 insertions(+), 11 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformerWithKey.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index aba48664..2ccef962 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -24,6 +24,7 @@ package com.bakdata.kafka; +import static java.util.Collections.emptyList; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -52,8 +53,11 @@ import org.apache.kafka.streams.kstream.Predicate; import org.apache.kafka.streams.kstream.Printed; import org.apache.kafka.streams.kstream.Produced; +import org.apache.kafka.streams.kstream.TransformerSupplier; import org.apache.kafka.streams.kstream.ValueMapper; import org.apache.kafka.streams.kstream.ValueMapperWithKey; +import org.apache.kafka.streams.kstream.ValueTransformerSupplier; +import org.apache.kafka.streams.kstream.ValueTransformerWithKeySupplier; import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.apache.kafka.streams.processor.api.FixedKeyRecord; import org.apache.kafka.streams.processor.api.ProcessorSupplier; @@ -3705,7 +3709,745 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO transform + @Test + void shouldTransform() { + final TransformerSupplier> transformer = + () -> new SimpleTransformer<>() { + + @Override + public KeyValue transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return KeyValue.pair("baz", "qux"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.transform(transformer); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformNamed() { + final TransformerSupplier> transformer = + () -> new SimpleTransformer<>() { + + @Override + public KeyValue transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return KeyValue.pair("baz", "qux"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.transform(transformer, Named.as("transform")); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformUsingStore() { + final TransformerSupplier> transformer = + () -> new SimpleTransformer<>() { + + @Override + public KeyValue transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.transform(transformer, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformNamedUsingStore() { + final TransformerSupplier> transformer = + () -> new SimpleTransformer<>() { + + @Override + public KeyValue transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.transform(transformer, Named.as("transform"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldFlatTransform() { + final TransformerSupplier>> transformer = + () -> new SimpleTransformer<>() { + + @Override + public Iterable> transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return List.of(KeyValue.pair("baz", "qux")); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.flatTransform(transformer); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatTransformNamed() { + final TransformerSupplier>> transformer = + () -> new SimpleTransformer<>() { + + @Override + public Iterable> transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return List.of(KeyValue.pair("baz", "qux")); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = + input.flatTransform(transformer, Named.as("flatTransform")); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("baz") + .hasValue("qux") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatTransformUsingStore() { + final TransformerSupplier>> transformer = + () -> new SimpleTransformer<>() { + + @Override + public Iterable> transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.flatTransform(transformer, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldFlatTransformNamedUsingStore() { + final TransformerSupplier>> transformer = + () -> new SimpleTransformer<>() { + + @Override + public Iterable> transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.flatTransform(transformer, Named.as("flatTransform"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValues() { + final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String value) { + if ("bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.transformValues(transformer); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesNamed() { + final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String value) { + if ("bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.transformValues(transformer, Named.as("transform")); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesUsingStore() { + final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(value, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.transformValues(transformer, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("bar")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValuesNamedUsingStore() { + final ValueTransformerSupplier transformer = () -> new SimpleValueTransformer<>() { + + @Override + public String transform(final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(value, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.transformValues(transformer, Named.as("transform"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("bar")).isEqualTo("bar"); + } + } + + @Test + void shouldFlatTransformValues() { + final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { + + @Override + public Iterable transform(final String value) { + if ("bar".equals(value)) { + return List.of("baz"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.flatTransformValues(transformer); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatTransformValuesNamed() { + final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { + + @Override + public Iterable transform(final String value) { + if ("bar".equals(value)) { + return List.of("baz"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = + input.flatTransformValues(transformer, Named.as("flatTransform")); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatTransformValuesUsingStore() { + final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { + + @Override + public Iterable transform(final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(value, value); + return emptyList(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.flatTransformValues(transformer, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("bar")).isEqualTo("bar"); + } + } + + @Test + void shouldFlatTransformValuesNamedUsingStore() { + final ValueTransformerSupplier> transformer = () -> new SimpleValueTransformer<>() { + + @Override + public Iterable transform(final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(value, value); + return emptyList(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.flatTransformValues(transformer, Named.as("flatTransform"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("bar")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValuesWithKey() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public String transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.transformValues(transformer); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesWithKeyNamed() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public String transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return "baz"; + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.transformValues(transformer, Named.as("transform")); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldTransformValuesWithKeyUsingStore() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public String transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.transformValues(transformer, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldTransformValuesWithKeyNamedUsingStore() { + final ValueTransformerWithKeySupplier transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public String transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return null; + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.transformValues(transformer, Named.as("transform"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldFlatTransformValuesWithKey() { + final ValueTransformerWithKeySupplier> transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public Iterable transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return List.of("baz"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = input.flatTransformValues(transformer); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatTransformValuesWithKeyNamed() { + final ValueTransformerWithKeySupplier> transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public Iterable transform(final String key, final String value) { + if ("foo".equals(key) && "bar".equals(value)) { + return List.of("baz"); + } + throw new UnsupportedOperationException(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX transformed = + input.flatTransformValues(transformer, Named.as("flatTransform")); + transformed.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldFlatTransformValuesWithKeyUsingStore() { + final ValueTransformerWithKeySupplier> transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public Iterable transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return emptyList(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.flatTransformValues(transformer, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldFlatTransformValuesWithKeyNamedUsingStore() { + final ValueTransformerWithKeySupplier> transformer = + () -> new SimpleValueTransformerWithKey<>() { + + @Override + public Iterable transform(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + return emptyList(); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.flatTransformValues(transformer, Named.as("flatTransform"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + //TODO process (old) } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java index e2289afa..fc5268b7 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KTableXTest.java @@ -1035,7 +1035,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValues() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1066,7 +1066,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesNamed() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1097,7 +1097,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesUsingMaterialized() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1135,7 +1135,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesNamedUsingMaterialized() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1174,7 +1174,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesUsingStore() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1206,7 +1206,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesNamedUsingStore() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1239,7 +1239,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesUsingStoreAndMaterialized() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { @@ -1277,7 +1277,7 @@ public void buildTopology(final TopologyBuilder builder) { @Test void shouldTransformValuesNamedUsingStoreAndMaterialized() { final ValueTransformerWithKeySupplier transformer = - () -> new SimpleValueTransformer<>() { + () -> new SimpleValueTransformerWithKey<>() { @Override public String transform(final String readOnlyKey, final String value) { diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java new file mode 100644 index 00000000..d458c68c --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleTransformer.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.kstream.Transformer; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.StateStore; + +abstract class SimpleTransformer implements Transformer { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + @Override + public void close() { + // do nothing + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java index e7d848a8..61d7c4ad 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformer.java @@ -24,11 +24,11 @@ package com.bakdata.kafka; -import org.apache.kafka.streams.kstream.ValueTransformerWithKey; +import org.apache.kafka.streams.kstream.ValueTransformer; import org.apache.kafka.streams.processor.ProcessorContext; import org.apache.kafka.streams.processor.StateStore; -abstract class SimpleValueTransformer implements ValueTransformerWithKey { +abstract class SimpleValueTransformer implements ValueTransformer { private ProcessorContext context = null; @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformerWithKey.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformerWithKey.java new file mode 100644 index 00000000..5ade8948 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleValueTransformerWithKey.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.kstream.ValueTransformerWithKey; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.StateStore; + +abstract class SimpleValueTransformerWithKey implements ValueTransformerWithKey { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + @Override + public void close() { + // do nothing + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + +} From a1849d222c17cbc13a2f08c64dd15acc124c1eb2 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 10:08:41 +0100 Subject: [PATCH 59/72] Add tests --- .../java/com/bakdata/kafka/KStreamXTest.java | 60 ++++++++++++++++++- .../bakdata/kafka/SimpleLegacyProcessor.java | 48 +++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 2ccef962..0bd55297 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -4448,6 +4448,64 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO process (old) + @Test + void shouldLegacyProcess() { + final org.apache.kafka.streams.processor.ProcessorSupplier processor = + () -> new SimpleLegacyProcessor<>() { + + @Override + public void process(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.process(processor, "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } + + @Test + void shouldLegacyProcessNamed() { + final org.apache.kafka.streams.processor.ProcessorSupplier processor = + () -> new SimpleLegacyProcessor<>() { + + @Override + public void process(final String key, final String value) { + final KeyValueStore store = this.getStateStore("my-store"); + store.put(key, value); + } + }; + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final StoreBuilder> store = builder.stores() + .keyValueStoreBuilder(Stores.inMemoryKeyValueStore("my-store"), Preconfigured.defaultSerde(), + Preconfigured.defaultSerde()); + builder.addStateStore(store); + final KStreamX input = builder.stream("input"); + input.process(processor, Named.as("process"), "my-store"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input().add("foo", "bar"); + final KeyValueStore store = + topology.getTestDriver().getKeyValueStore("my-store"); + this.softly.assertThat(store.get("foo")).isEqualTo("bar"); + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java new file mode 100644 index 00000000..951ad25a --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/SimpleLegacyProcessor.java @@ -0,0 +1,48 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import org.apache.kafka.streams.processor.Processor; +import org.apache.kafka.streams.processor.ProcessorContext; +import org.apache.kafka.streams.processor.StateStore; + +abstract class SimpleLegacyProcessor implements Processor { + private ProcessorContext context = null; + + @Override + public void init(final ProcessorContext context) { + this.context = context; + } + + @Override + public void close() { + // do nothing + } + + protected S getStateStore(final String name) { + return this.context.getStateStore(name); + } + +} From 9fc61bc6e09781fa5c08be0f965dbb7ca5efd27c Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 11:27:48 +0100 Subject: [PATCH 60/72] Add tests --- .../kafka/util/TopologyInformation.java | 12 +- .../java/com/bakdata/kafka/BranchedXTest.java | 129 +++++++ .../java/com/bakdata/kafka/ConsumedXTest.java | 303 +++++++++++++++++ .../java/com/bakdata/kafka/ProducedXTest.java | 251 ++++++++++++++ .../com/bakdata/kafka/RepartitionedXTest.java | 321 ++++++++++++++++++ 5 files changed, 1013 insertions(+), 3 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java index 30b75408..237099ab 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +31,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.kafka.streams.Topology; +import org.apache.kafka.streams.TopologyDescription; import org.apache.kafka.streams.TopologyDescription.Node; import org.apache.kafka.streams.TopologyDescription.Processor; import org.apache.kafka.streams.TopologyDescription.Sink; @@ -69,13 +70,18 @@ public TopologyInformation(final Topology topology, final String streamsId) { this.streamsId = streamsId; } - private static List getNodes(final Topology topology) { - return topology.describe().subtopologies() + public static List getNodes(final TopologyDescription description) { + return description.subtopologies() .stream() .flatMap(subtopology -> subtopology.nodes().stream()) .collect(Collectors.toList()); } + private static List getNodes(final Topology topology) { + final TopologyDescription description = topology.describe(); + return getNodes(description); + } + private static Stream getAllSources(final Collection nodes) { return nodes.stream() .filter(Source.class::isInstance) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java new file mode 100644 index 00000000..c6d407bc --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java @@ -0,0 +1,129 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.util.Map; +import java.util.function.Function; +import org.apache.kafka.streams.kstream.Named; +import org.junit.jupiter.api.Test; + +class BranchedXTest { + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split(Named.as("split-")) + .defaultBranch(BranchedX.as("default")); + branches.get("split-default").toOutputTopic(); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split(Named.as("split-")) + .defaultBranch(BranchedX.withFunction(Function.identity()).withName("default")); + branches.get("split-default").toOutputTopic(); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseConsumer() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .defaultBranch(BranchedX.withConsumer(KStreamX::toOutputTopic)); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseFunction() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final Map> branches = input.split() + .defaultBranch(BranchedX.withFunction(s -> s.mapValues(v -> "baz"))); + branches.values().iterator().next().toOutputTopic(); + } + }; + try (final TestTopology topology = app.startApp(StreamsTopicConfig.builder() + .outputTopic("output") + .build())) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("baz") + .expectNoMoreRecord(); + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java new file mode 100644 index 00000000..6f22be65 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -0,0 +1,303 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; +import java.util.List; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.Topology.AutoOffsetReset; +import org.apache.kafka.streams.TopologyDescription.Node; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class ConsumedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + input.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("foo") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.as("stream").withKeySerde(Serdes.Long())); + input.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("foo") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + input.to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.as("stream").withValueSerde(Serdes.Long())); + input.to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.as("stream")); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); + this.softly.assertThat(nodes) + .extracting(Node::name) + .contains("stream"); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + ConsumedX.keySerde(Preconfigured.defaultSerde()).withName("stream")); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); + this.softly.assertThat(nodes) + .extracting(Node::name) + .contains("stream"); + } + } + + @Test + void shouldUseTimestampExtractor() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + ConsumedX.with((record, partitionTime) -> 1L)); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + final List> records = topology.streamOutput().toList(); + this.softly.assertThat(records) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + this.softly.assertThat(outputRecord.timestamp()).isEqualTo(1L); + }); + } + } + + @Test + void shouldUseTimestampExtractorModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.as("stream") + .withTimestampExtractor((record, partitionTime) -> 1L)); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + final List> records = topology.streamOutput().toList(); + this.softly.assertThat(records) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + this.softly.assertThat(outputRecord.timestamp()).isEqualTo(1L); + }); + } + } + + @Test + void shouldUseOffsetResetPolicy() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(AutoOffsetReset.LATEST)); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test existing records. TestDriver cannot have existing records + } + } + + @Test + void shouldUseOffsetResetPolicyModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", + ConsumedX.as("stream").withOffsetResetPolicy(AutoOffsetReset.LATEST)); + input.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test existing records. TestDriver cannot have existing records + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java new file mode 100644 index 00000000..e0d578ab --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java @@ -0,0 +1,251 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; +import java.util.List; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.TopologyDescription.Node; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class ProducedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + input.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("foo") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + input.to("output", ProducedX.as("output").withKeySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("foo") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + input.to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + input.to("output", ProducedX.as("output").withValueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + input.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.to("output", ProducedX.as("output")); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); + this.softly.assertThat(nodes) + .extracting(Node::name) + .contains("output"); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.to("output", ProducedX.keySerde(Preconfigured.defaultSerde()).withName("output")); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); + this.softly.assertThat(nodes) + .extracting(Node::name) + .contains("output"); + } + } + + @Test + void shouldUseStreamPartitioner() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.to("output", ProducedX.streamPartitioner((topic, key, value, numPartitions) -> 1)); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } + + @Test + void shouldUseStreamPartitionerModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + input.to("output", ProducedX.as("output") + .withStreamPartitioner((topic, key, value, numPartitions) -> 1)); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java new file mode 100644 index 00000000..caaf34b7 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java @@ -0,0 +1,321 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static org.assertj.core.api.InstanceOfAssertFactories.type; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; +import java.util.List; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.TopologyDescription.Node; +import org.apache.kafka.streams.TopologyDescription.Sink; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class RepartitionedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX repartitioned = input.repartition(RepartitionedX.keySerde(Serdes.Long())); + repartitioned.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("foo") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX repartitioned = + input.repartition(RepartitionedX.as("repartition").withKeySerde(Serdes.Long())); + repartitioned.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("foo") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX repartitioned = + input.repartition(RepartitionedX.valueSerde(Serdes.Long())); + repartitioned.to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX repartitioned = + input.repartition(RepartitionedX.as("repartition").withValueSerde(Serdes.Long())); + repartitioned.to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX repartitioned = + input.repartition(RepartitionedX.with(Serdes.Long(), Serdes.Long())); + repartitioned.repartition(RepartitionedX.keySerde(Serdes.Long())) + .to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = input.repartition(RepartitionedX.as("repartition")); + repartitioned.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); + this.softly.assertThat(nodes) + .anySatisfy(node -> this.softly.assertThat(node) + .asInstanceOf(type(Sink.class)) + .satisfies( + sink -> this.softly.assertThat(sink.topic()).isEqualTo("repartition-repartition"))); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = input.repartition( + RepartitionedX.keySerde(Preconfigured.defaultSerde()).withName("repartition")); + repartitioned.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); + this.softly.assertThat(nodes) + .anySatisfy(node -> this.softly.assertThat(node) + .asInstanceOf(type(Sink.class)) + .satisfies( + sink -> this.softly.assertThat(sink.topic()).isEqualTo("repartition-repartition"))); + } + } + + @Test + void shouldUseStreamPartitioner() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = + input.repartition(RepartitionedX.streamPartitioner((topic, key, value, numPartitions) -> 1)); + repartitioned.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } + + @Test + void shouldUseStreamPartitionerModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = input.repartition( + RepartitionedX.as("repartition") + .withStreamPartitioner((topic, key, value, numPartitions) -> 1)); + repartitioned.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } + + @Test + void shouldUseNumberOfPartitions() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = input.repartition(RepartitionedX.numberOfPartitions(2)); + repartitioned.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } + + @Test + void shouldUseNumberOfPartitionsModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KStreamX repartitioned = + input.repartition(RepartitionedX.as("repartition").withNumberOfPartitions(2)); + repartitioned.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } +} From 5761ed681affeada396f5b510763ff320555b03f Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 12:11:43 +0100 Subject: [PATCH 61/72] Add tests --- .../kafka/util/TopologyInformation.java | 60 +++-- .../java/com/bakdata/kafka/ConsumedXTest.java | 8 +- .../java/com/bakdata/kafka/GroupedXTest.java | 226 ++++++++++++++++++ .../java/com/bakdata/kafka/ProducedXTest.java | 9 +- .../com/bakdata/kafka/RepartitionedXTest.java | 23 +- .../bakdata/kafka/TestTopologyFactory.java | 6 + 6 files changed, 290 insertions(+), 42 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/GroupedXTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java index 237099ab..19393fc2 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java @@ -66,40 +66,53 @@ public class TopologyInformation { * @param streamsId unique app id to represent auto-created topics */ public TopologyInformation(final Topology topology, final String streamsId) { - this.nodes = getNodes(topology); + this(topology.describe(), streamsId); + } + + /** + * Create a new TopologyInformation of a topology and the unique app id + * + * @param description topology description to extract nodes from + * @param streamsId unique app id to represent auto-created topics + */ + public TopologyInformation(final TopologyDescription description, final String streamsId) { + this.nodes = getNodes(description); this.streamsId = streamsId; } - public static List getNodes(final TopologyDescription description) { + private static List getNodes(final TopologyDescription description) { return description.subtopologies() .stream() .flatMap(subtopology -> subtopology.nodes().stream()) .collect(Collectors.toList()); } - private static List getNodes(final Topology topology) { - final TopologyDescription description = topology.describe(); - return getNodes(description); + private static Stream getAllSubscriptions(final Collection nodes) { + return getAllSources(nodes) + .map(TopologyInformation::toSubscription); } - private static Stream getAllSources(final Collection nodes) { + private static Stream getAllSources(final Collection nodes) { return nodes.stream() .filter(Source.class::isInstance) - .map(Source.class::cast) - .map(TopologyInformation::getAllSources); + .map(Source.class::cast); } - private static TopicSubscription getAllSources(final Source source) { + private static TopicSubscription toSubscription(final Source source) { final Set topicSet = source.topicSet(); return topicSet == null ? new PatternTopicSubscription(source.topicPattern()) : new DirectTopicSubscription(topicSet); } - private static Stream getAllSinks(final Collection nodes) { + private static Stream getAllTopics(final Collection nodes) { + return getAllSinks(nodes) + .map(Sink::topic); + } + + private static Stream getAllSinks(final Collection nodes) { return nodes.stream() .filter(Sink.class::isInstance) - .map(Sink.class::cast) - .map(Sink::topic); + .map(Sink.class::cast); } private static Stream getAllStores(final Collection nodes) { @@ -152,7 +165,7 @@ public List getInternalTopics() { * @return list of external sink topics */ public List getExternalSinkTopics() { - return getAllSinks(this.nodes) + return getAllTopics(this.nodes) .filter(this::isExternalTopic) .collect(Collectors.toList()); } @@ -165,7 +178,7 @@ public List getExternalSinkTopics() { */ public List getExternalSourceTopics(final Collection allTopics) { final List sinks = this.getExternalSinkTopics(); - return getAllSources(this.nodes) + return getAllSubscriptions(this.nodes) .map(subscription -> subscription.resolveTopics(allTopics)) .flatMap(Collection::stream) .filter(this::isExternalTopic) @@ -182,7 +195,7 @@ public List getExternalSourceTopics(final Collection allTopics) */ public List getIntermediateTopics(final Collection allTopics) { final List sinks = this.getExternalSinkTopics(); - return getAllSources(this.nodes) + return getAllSubscriptions(this.nodes) .map(subscription -> subscription.resolveTopics(allTopics)) .flatMap(Collection::stream) .filter(this::isExternalTopic) @@ -190,6 +203,21 @@ public List getIntermediateTopics(final Collection allTopics) { .collect(Collectors.toList()); } + public List getProcessors() { + return getAllProcessors(this.nodes) + .collect(Collectors.toList()); + } + + public List getSources() { + return getAllSources(this.nodes) + .collect(Collectors.toList()); + } + + public List getSinks() { + return getAllSinks(this.nodes) + .collect(Collectors.toList()); + } + private boolean isInternalTopic(final String topic) { if (topic.startsWith("KSTREAM-") || topic.startsWith("KTABLE-")) { return true; @@ -213,7 +241,7 @@ private boolean isExternalTopic(final String topic) { } private Stream getInternalSinks() { - return getAllSinks(this.nodes) + return getAllTopics(this.nodes) .filter(this::isInternalTopic) .flatMap(topic -> Seq.of(topic).concat(createPseudoTopics(topic))) .map(topic -> String.format("%s-%s", this.streamsId, topic)); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index 6f22be65..e5319771 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -175,8 +175,8 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); - this.softly.assertThat(nodes) + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getSources()) .extracting(Node::name) .contains("stream"); } @@ -200,8 +200,8 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); - this.softly.assertThat(nodes) + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getSources()) .extracting(Node::name) .contains("stream"); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/GroupedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/GroupedXTest.java new file mode 100644 index 00000000..236782d7 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/GroupedXTest.java @@ -0,0 +1,226 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; +import org.apache.kafka.common.serialization.Serdes; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class GroupedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey(GroupedX.keySerde(Serdes.Long())); + final KStreamX counted = grouped.count().toStream(); + counted.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey(GroupedX.as("grouped").withKeySerde(Serdes.Long())); + final KStreamX counted = grouped.count().toStream(); + counted.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .add(1L, "foo"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey(GroupedX.valueSerde(Serdes.Long())); + final KStreamX counted = grouped.count().toStream(); + counted.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey(GroupedX.as("grouped").withValueSerde(Serdes.Long())); + final KStreamX counted = grouped.count().toStream(); + counted.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey(GroupedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX counted = grouped.count().toStream(); + counted.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(1L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey(GroupedX.as("grouped")); + final KStreamX counted = grouped.count().toStream(); + counted.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getInternalTopics()) + .anySatisfy(topic -> this.softly.assertThat(topic).endsWith("grouped-repartition")); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.selectKey((k, v) -> k) + .groupByKey( + GroupedX.keySerde(Preconfigured.defaultSerde()).withName("grouped")); + final KStreamX counted = grouped.count().toStream(); + counted.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input() + .add("foo", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(1L) + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getInternalTopics()) + .anySatisfy(topic -> this.softly.assertThat(topic).endsWith("grouped-repartition")); + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java index e0d578ab..2c7b03f8 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java @@ -26,7 +26,6 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import com.bakdata.kafka.util.TopologyInformation; -import java.util.List; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.TopologyDescription.Node; import org.assertj.core.api.SoftAssertions; @@ -173,8 +172,8 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); - this.softly.assertThat(nodes) + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getSinks()) .extracting(Node::name) .contains("output"); } @@ -197,8 +196,8 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); - this.softly.assertThat(nodes) + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getSinks()) .extracting(Node::name) .contains("output"); } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java index caaf34b7..f12dd343 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java @@ -24,14 +24,9 @@ package com.bakdata.kafka; -import static org.assertj.core.api.InstanceOfAssertFactories.type; - import com.bakdata.fluent_kafka_streams_tests.TestTopology; import com.bakdata.kafka.util.TopologyInformation; -import java.util.List; import org.apache.kafka.common.serialization.Serdes; -import org.apache.kafka.streams.TopologyDescription.Node; -import org.apache.kafka.streams.TopologyDescription.Sink; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; @@ -186,12 +181,9 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); - this.softly.assertThat(nodes) - .anySatisfy(node -> this.softly.assertThat(node) - .asInstanceOf(type(Sink.class)) - .satisfies( - sink -> this.softly.assertThat(sink.topic()).isEqualTo("repartition-repartition"))); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getInternalTopics()) + .anySatisfy(topic -> this.softly.assertThat(topic).endsWith("repartition-repartition")); } } @@ -214,12 +206,9 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("bar") .expectNoMoreRecord(); - final List nodes = TopologyInformation.getNodes(topology.getTopologyDescription()); - this.softly.assertThat(nodes) - .anySatisfy(node -> this.softly.assertThat(node) - .asInstanceOf(type(Sink.class)) - .satisfies( - sink -> this.softly.assertThat(sink.topic()).isEqualTo("repartition-repartition"))); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getInternalTopics()) + .anySatisfy(topic -> this.softly.assertThat(topic).endsWith("repartition-repartition")); } } diff --git a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java index 883a027b..77b0e90c 100644 --- a/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java +++ b/streams-bootstrap-test/src/main/java/com/bakdata/kafka/TestTopologyFactory.java @@ -28,6 +28,7 @@ import com.bakdata.fluent_kafka_streams_tests.TestTopology; import com.bakdata.fluent_kafka_streams_tests.junit5.TestTopologyExtension; +import com.bakdata.kafka.util.TopologyInformation; import io.confluent.kafka.schemaregistry.SchemaProvider; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient; import io.confluent.kafka.schemaregistry.client.SchemaRegistryClientFactory; @@ -94,6 +95,11 @@ public static Configurator createConfigurator(final TestTopology testTopol return new Configurator(testTopology.getProperties()); } + public static TopologyInformation getTopologyInformation(final TestTopology topology) { + final ImprovedStreamsConfig streamsConfig = new ImprovedStreamsConfig(topology.getStreamsConfig()); + return new TopologyInformation(topology.getTopologyDescription(), streamsConfig.getAppId()); + } + /** * Create a new Kafka Streams config suitable for test environments. This includes setting the following * parameters in addition to {@link #createStreamsTestConfig()}: From 0b6dae5170b2347befa9db91414907be62d31c91 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 12:50:38 +0100 Subject: [PATCH 62/72] Add tests --- .../java/com/bakdata/kafka/JoinedXTest.java | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java new file mode 100644 index 00000000..c4c57f34 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java @@ -0,0 +1,319 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.time.Duration; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.state.Stores; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class JoinedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KTableX table = builder.table("table_input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX joined = + stream.join(table, (v1, v2) -> v1 + v2, JoinedX.keySerde(Serdes.Long())); + joined.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.Long()) + .add(1L, "baz"); + topology.input("input") + .withKeySerde(Serdes.Long()) + .add(1L, "bar"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KTableX table = builder.table("table_input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX joined = stream.join(table, (v1, v2) -> v1 + v2, + JoinedX.as("join").withKeySerde(Serdes.Long())); + joined.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.Long()) + .add(1L, "baz"); + topology.input("input") + .withKeySerde(Serdes.Long()) + .add(1L, "bar"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KTableX table = builder.table("table_input"); + final KStreamX joined = + stream.join(table, (v1, v2) -> v1 + v2, JoinedX.valueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("1baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KTableX table = builder.table("table_input"); + final KStreamX joined = stream.join(table, (v1, v2) -> v1 + v2, + JoinedX.as("join").withValueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("1baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseOtherValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KTableX table = builder.table("table_input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX joined = + stream.join(table, (v1, v2) -> v1 + v2, JoinedX.otherValueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar1") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseOtherValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KTableX table = builder.table("table_input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX joined = stream.join(table, (v1, v2) -> v1 + v2, + JoinedX.as("join").withOtherValueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar1") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KTableX table = + builder.table("table_input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX joined = + stream.join(table, Long::sum, JoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KTableX table = builder.table("table_input"); + final KStreamX joined = stream.join(table, (v1, v2) -> v1 + v2, JoinedX.as("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KTableX table = builder.table("table_input"); + final KStreamX joined = stream.join(table, (v1, v2) -> v1 + v2, + JoinedX.keySerde(Preconfigured.defaultSerde()).withName("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseGracePeriodModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KTableX table = builder.table("table_input", + MaterializedX.as( + Stores.persistentVersionedKeyValueStore("store", Duration.ofSeconds(1L)))); + final KStreamX joined = stream.join(table, (v1, v2) -> v1 + v2, + JoinedX.as("join").withGracePeriod(Duration.ofSeconds(1L))); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("table_input") + .at(0L) + .add("foo", "baz"); + topology.input("input") + .at(1000L) //advance stream time + .add("", ""); + topology.input("table_input") + .at(1000L) + .add("foo", "qux"); + topology.input("input") + .at(0L) + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } +} From 3d9ba36138a7897e808a9c58e8ce7eb9fe98ace2 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 13:49:09 +0100 Subject: [PATCH 63/72] Add tests --- .../kafka/util/TopologyInformation.java | 8 +- .../java/com/bakdata/kafka/JoinedXTest.java | 21 +- .../com/bakdata/kafka/StreamJoinedXTest.java | 574 ++++++++++++++++++ 3 files changed, 596 insertions(+), 7 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java index 19393fc2..1d9fd5d4 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java @@ -117,7 +117,8 @@ private static Stream getAllSinks(final Collection nodes) { private static Stream getAllStores(final Collection nodes) { return getAllProcessors(nodes) - .flatMap(processor -> processor.stores().stream()); + .flatMap(processor -> processor.stores().stream()) + .distinct(); } private static Stream getAllProcessors(final Collection nodes) { @@ -203,6 +204,11 @@ public List getIntermediateTopics(final Collection allTopics) { .collect(Collectors.toList()); } + public List getStores() { + return getAllStores(this.nodes) + .collect(Collectors.toList()); + } + public List getProcessors() { return getAllProcessors(this.nodes) .collect(Collectors.toList()); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java index c4c57f34..79981a0d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/JoinedXTest.java @@ -25,8 +25,10 @@ package com.bakdata.kafka; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; import java.time.Duration; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.TopologyDescription.Node; import org.apache.kafka.streams.state.Stores; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; @@ -254,6 +256,10 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("barbaz") .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getProcessors()) + .extracting(Node::name) + .contains("join"); } } @@ -279,6 +285,10 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("barbaz") .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getProcessors()) + .extracting(Node::name) + .contains("join"); } } @@ -300,15 +310,14 @@ public void buildTopology(final TopologyBuilder builder) { topology.input("table_input") .at(0L) .add("foo", "baz"); - topology.input("input") - .at(1000L) //advance stream time - .add("", ""); - topology.input("table_input") - .at(1000L) - .add("foo", "qux"); topology.input("input") .at(0L) .add("foo", "bar"); + topology.streamOutput() + .expectNoMoreRecord(); + topology.input("input") + .at(1000L) //advance stream time + .add("", ""); topology.streamOutput() .expectNextRecord() .hasKey("foo") diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java new file mode 100644 index 00000000..bb5fa771 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java @@ -0,0 +1,574 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static java.util.Collections.emptyMap; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.streams.TopologyDescription.Node; +import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.JoinWindows; +import org.apache.kafka.streams.kstream.Materialized.StoreType; +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.state.Stores; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class StreamJoinedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX otherInput = + builder.stream("other_input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX joined = + stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Serdes.Long())); + joined.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .add(1L, "baz"); + topology.input("input") + .withKeySerde(Serdes.Long()) + .add(1L, "bar"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX otherInput = + builder.stream("other_input", ConsumedX.keySerde(Serdes.Long())); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.as("join").withKeySerde(Serdes.Long())); + joined.to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .add(1L, "baz"); + topology.input("input") + .withKeySerde(Serdes.Long()) + .add(1L, "bar"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("barbaz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = + stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.valueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("1baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.as("join").withValueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("1baz") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseOtherValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = + builder.stream("other_input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX joined = + stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.otherValueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar1") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseOtherValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = + builder.stream("other_input", ConsumedX.valueSerde(Serdes.Long())); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.as("join").withOtherValueSerde(Serdes.Long())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withValueSerde(Serdes.Long()) + .add("foo", 1L); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar1") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = + builder.stream("input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX table = + builder.stream("other_input", ConsumedX.with(Serdes.Long(), Serdes.Long())); + final KStreamX joined = + stream.join(table, Long::sum, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.with(Serdes.Long(), Serdes.Long(), Serdes.Long())); + joined.to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 3L); + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(5L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseStoreName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.as("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getStores()) + .filteredOn(store -> store.startsWith("join")) + .hasSize(2); + } + } + + @Test + void shouldUseStoreNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + .withStoreName("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getStores()) + .filteredOn(store -> store.startsWith("join")) + .hasSize(2); + } + } + + @Test + void shouldUseNameModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()).withName("join")); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getProcessors()) + .extracting(Node::name) + .filteredOn(processor -> processor.startsWith("join")) + .hasSize(5); + } + } + + @Test + void shouldUseDslStoreSupplier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.with(StoreType.IN_MEMORY)); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyTestDriver testDriver = topology.getTestDriver(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + final List stateStores = stores.stream() + .map(testDriver::getWindowStore) + .collect(Collectors.toList()); + this.softly.assertThat(stateStores) + .allSatisfy(stateStore -> this.softly.assertThat(stateStore.persistent()).isFalse()); + } + } + + @Test + void shouldUseDslStoreSupplierModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + .withDslStoreSuppliers(StoreType.IN_MEMORY)); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyTestDriver testDriver = topology.getTestDriver(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + final List stateStores = stores.stream() + .map(testDriver::getWindowStore) + .collect(Collectors.toList()); + this.softly.assertThat(stateStores) + .allSatisfy(stateStore -> this.softly.assertThat(stateStore.persistent()).isFalse()); + } + } + + @Test + void shouldUseStoreSuppliers() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.with( + Stores.inMemoryWindowStore("this-store", Duration.ofMinutes(2L), Duration.ofMinutes(2L), + true), Stores.inMemoryWindowStore("other-store", Duration.ofMinutes(2L), + Duration.ofMinutes(2L), true))); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + this.softly.assertThat(stores) + .contains("this-store", "other-store"); + } + } + + @Test + void shouldUseThisStoreSuppliersModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + .withThisStoreSupplier(Stores.inMemoryWindowStore("store", Duration.ofMinutes(2L), + Duration.ofMinutes(2L), true))); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + this.softly.assertThat(stores) + .contains("store"); + } + } + + @Test + void shouldUseOtherStoreSuppliersModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + .withOtherStoreSupplier(Stores.inMemoryWindowStore("store", Duration.ofMinutes(2L), + Duration.ofMinutes(2L), true))); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + this.softly.assertThat(stores) + .contains("store"); + } + } + + @Test + void shouldDisableLogging() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + .withLoggingDisabled()); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } + + @Test + void shouldEnableLogging() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX stream = builder.stream("input"); + final KStreamX otherInput = builder.stream("other_input"); + final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, + JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), + StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + .withLoggingEnabled(emptyMap())); + joined.to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("other_input") + .add("foo", "baz"); + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("barbaz") + .expectNoMoreRecord(); + // TODO test partition. TestDriver does not expose it + } + } +} From 7a368a71878cd3d76ccafb44e338712550b4e648 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 18:51:33 +0100 Subject: [PATCH 64/72] Add tests --- .../com/bakdata/kafka/MaterializedXTest.java | 440 ++++++++++++++++++ .../com/bakdata/kafka/StreamJoinedXTest.java | 15 +- 2 files changed, 447 insertions(+), 8 deletions(-) create mode 100644 streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java new file mode 100644 index 00000000..89cba097 --- /dev/null +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java @@ -0,0 +1,440 @@ +/* + * MIT License + * + * Copyright (c) 2025 bakdata + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package com.bakdata.kafka; + +import static java.util.Collections.emptyMap; + +import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.util.TopologyInformation; +import java.time.Duration; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.utils.Bytes; +import org.apache.kafka.streams.TopologyTestDriver; +import org.apache.kafka.streams.kstream.Materialized.StoreType; +import org.apache.kafka.streams.kstream.SessionWindows; +import org.apache.kafka.streams.kstream.TimeWindows; +import org.apache.kafka.streams.kstream.Windowed; +import org.apache.kafka.streams.processor.StateStore; +import org.apache.kafka.streams.state.KeyValueStore; +import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.internals.CachingKeyValueStore; +import org.apache.kafka.streams.state.internals.MeteredTimestampedKeyValueStore; +import org.apache.kafka.streams.state.internals.WrappedStateStore; +import org.assertj.core.api.InstanceOfAssertFactories; +import org.assertj.core.api.SoftAssertions; +import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; +import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(SoftAssertionsExtension.class) +class MaterializedXTest { + + @InjectSoftAssertions + private SoftAssertions softly; + + @Test + void shouldUseKeySerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", MaterializedX.keySerde(Serdes.Long())); + table.toStream().to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .add(1L, "bar"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseKeySerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>as("store") + .withKeySerde(Serdes.Long())); + table.toStream().to("output", ProducedX.keySerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .add(1L, "bar"); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue("bar") + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerde() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", MaterializedX.valueSerde(Serdes.Long())); + table.toStream().to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withValueSerde(Serdes.Long()) + .add("foo", 2L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseValueSerdeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>as("store") + .withValueSerde(Serdes.Long())); + table.toStream().to("output", ProducedX.valueSerde(Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withValueSerde(Serdes.Long()) + .add("foo", 2L); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo") + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseSerdes() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = + builder.table("input", MaterializedX.with(Serdes.Long(), Serdes.Long())); + table.toStream().to("output", ProducedX.with(Serdes.Long(), Serdes.Long())); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .add(1L, 2L); + topology.streamOutput() + .withKeySerde(Serdes.Long()) + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey(1L) + .hasValue(2L) + .expectNoMoreRecord(); + } + } + + @Test + void shouldUseName() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", MaterializedX.as("store")); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + this.softly.assertThat(information.getStores()) + .contains("store"); + } + } + + @Test + void shouldUseStoreType() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", MaterializedX.as(StoreType.IN_MEMORY)); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final TopologyTestDriver testDriver = topology.getTestDriver(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + final List stateStores = stores.stream() + .map(testDriver::getKeyValueStore) + .collect(Collectors.toList()); + this.softly.assertThat(stateStores) + .allSatisfy(stateStore -> this.softly.assertThat(stateStore.persistent()).isFalse()); + } + } + + @Test + void shouldUseStoreTypeModifier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>keySerde( + Preconfigured.defaultSerde()).withStoreType(StoreType.IN_MEMORY)); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final TopologyTestDriver testDriver = topology.getTestDriver(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + final List stateStores = stores.stream() + .map(testDriver::getKeyValueStore) + .collect(Collectors.toList()); + this.softly.assertThat(stateStores) + .allSatisfy(stateStore -> this.softly.assertThat(stateStore.persistent()).isFalse()); + } + } + + @Test + void shouldUseKeyValueStoreSupplier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.as(Stores.inMemoryKeyValueStore("store"))); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + this.softly.assertThat(stores) + .contains("store"); + } + } + + @Test + void shouldUseWindowStoreSupplier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMinutes(1L))); + final KTableX, Long> counted = windowed.count(MaterializedX.as( + Stores.inMemoryWindowStore("store", Duration.ofMinutes(1L), Duration.ofMinutes(1L), false))); + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + this.softly.assertThat(stores) + .contains("store"); + } + } + + @Test + void shouldUseSessionStoreSupplier() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final SessionWindowedKStreamX windowed = + grouped.windowedBy(SessionWindows.ofInactivityGapWithNoGrace(Duration.ofMinutes(1L))); + final KTableX, Long> counted = windowed.count(MaterializedX.as( + Stores.inMemorySessionStore("store", Duration.ofMinutes(1L)))); + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNoMoreRecord(); + final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); + final List stores = information.getStores(); + this.softly.assertThat(stores) + .contains("store"); + } + } + + @Test + void shouldDisableLogging() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>keySerde( + Preconfigured.defaultSerde()).withLoggingDisabled()); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test topic existence. TestDriver does not expose it + } + } + + @Test + void shouldEnableLogging() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>keySerde( + Preconfigured.defaultSerde()).withLoggingEnabled(emptyMap())); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + // TODO test topic config. TestDriver does not expose it + } + } + + @Test + void shouldDisableCaching() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>as("store").withCachingDisabled()); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final TopologyTestDriver testDriver = topology.getTestDriver(); + this.softly.assertThat(testDriver.getTimestampedKeyValueStore("store")) + .asInstanceOf(InstanceOfAssertFactories.type(MeteredTimestampedKeyValueStore.class)) + .extracting(WrappedStateStore::wrapped) + .isNotInstanceOf(CachingKeyValueStore.class); + } + } + + @Test + void shouldEnableCaching() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KTableX table = builder.table("input", + MaterializedX.>as("store").withCachingEnabled()); + table.toStream().to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); + final TopologyTestDriver testDriver = topology.getTestDriver(); + this.softly.assertThat(testDriver.getTimestampedKeyValueStore("store")) + .asInstanceOf(InstanceOfAssertFactories.type(MeteredTimestampedKeyValueStore.class)) + .extracting(WrappedStateStore::wrapped) + .isInstanceOf(CachingKeyValueStore.class); + } + } +} diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java index bb5fa771..57909517 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java @@ -283,8 +283,7 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); this.softly.assertThat(information.getStores()) - .filteredOn(store -> store.startsWith("join")) - .hasSize(2); + .anySatisfy(store -> this.softly.assertThat(store).startsWith("join")); } } @@ -314,8 +313,7 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); this.softly.assertThat(information.getStores()) - .filteredOn(store -> store.startsWith("join")) - .hasSize(2); + .anySatisfy(store -> this.softly.assertThat(store).startsWith("join")); } } @@ -345,8 +343,7 @@ public void buildTopology(final TopologyBuilder builder) { final TopologyInformation information = TestTopologyFactory.getTopologyInformation(topology); this.softly.assertThat(information.getProcessors()) .extracting(Node::name) - .filteredOn(processor -> processor.startsWith("join")) - .hasSize(5); + .anySatisfy(processor -> this.softly.assertThat(processor).startsWith("join")); } } @@ -540,7 +537,7 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("barbaz") .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + // TODO test topic existence. TestDriver does not expose it } } @@ -568,7 +565,9 @@ public void buildTopology(final TopologyBuilder builder) { .hasKey("foo") .hasValue("barbaz") .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + // TODO test topic config. TestDriver does not expose it } } + + //TODO retention } From 4b45b597f05a01503ca82f73e7fa26f28e4a8995 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 20:01:43 +0100 Subject: [PATCH 65/72] Add tests --- .../kafka/integration/StreamsCleanUpTest.java | 6 +- .../com/bakdata/kafka/util/TopicClient.java | 36 +++ .../java/com/bakdata/kafka/ConsumedXTest.java | 92 +++++-- .../java/com/bakdata/kafka/DoubleApp.java | 5 +- .../com/bakdata/kafka/MaterializedXTest.java | 114 +++++++-- .../java/com/bakdata/kafka/ProducedXTest.java | 108 ++++++-- .../com/bakdata/kafka/RepartitionedXTest.java | 236 ++++++++++++++---- .../com/bakdata/kafka/StreamJoinedXTest.java | 125 +++++++--- .../java/com/bakdata/kafka/StringApp.java | 22 +- .../java/com/bakdata/kafka/TestHelper.java | 43 +++- .../integration/StreamsCleanUpRunnerTest.java | 141 +++++------ .../kafka/integration/StreamsRunnerTest.java | 31 +-- .../bakdata/kafka/util/TopicClientTest.java | 15 ++ .../java/com/bakdata/kafka/KafkaTest.java | 44 ++-- 14 files changed, 745 insertions(+), 273 deletions(-) diff --git a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpTest.java b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpTest.java index 200af6f0..82fc0bb2 100644 --- a/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpTest.java +++ b/streams-bootstrap-cli/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpTest.java @@ -83,7 +83,7 @@ void shouldClean() { this.runAndAssertContent(expectedValues, "All entries are once in the input topic after the 1st run", app); // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(app.createExecutableApp()); + awaitClosed(app.createExecutableApp()); app.clean(); try (final ImprovedAdminClient admin = testClient.admin()) { @@ -119,7 +119,7 @@ void shouldReset() { this.runAndAssertContent(expectedValues, "All entries are once in the input topic after the 1st run", app); // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(app.createExecutableApp()); + awaitClosed(app.createExecutableApp()); app.reset(); try (final ImprovedAdminClient admin = testClient.admin()) { @@ -158,7 +158,7 @@ private void runApp(final KafkaStreamsApplication app) { // run in Thread because the application blocks indefinitely new Thread(app).start(); // Wait until stream application has consumed all data - this.awaitProcessing(app.createExecutableApp()); + awaitProcessing(app.createExecutableApp()); } private CloseFlagApp createCloseFlagApplication() { diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopicClient.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopicClient.java index 7118b074..6789c874 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopicClient.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopicClient.java @@ -36,11 +36,14 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.function.Function; +import java.util.stream.Collectors; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.admin.Admin; import org.apache.kafka.clients.admin.AdminClient; +import org.apache.kafka.clients.admin.Config; +import org.apache.kafka.clients.admin.ConfigEntry; import org.apache.kafka.clients.admin.ListOffsetsResult.ListOffsetsResultInfo; import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.clients.admin.OffsetSpec; @@ -48,6 +51,8 @@ import org.apache.kafka.common.KafkaFuture; import org.apache.kafka.common.TopicPartition; import org.apache.kafka.common.TopicPartitionInfo; +import org.apache.kafka.common.config.ConfigResource; +import org.apache.kafka.common.config.ConfigResource.Type; import org.jooq.lambda.Seq; /** @@ -85,6 +90,10 @@ private static KafkaAdminException failedToRetrieveTopicDescription(final String return new KafkaAdminException("Failed to retrieve description of topic " + topicName, e); } + private static KafkaAdminException failedToRetrieveTopicConfig(final String topicName, final Throwable e) { + return new KafkaAdminException("Failed to retrieve config of topic " + topicName, e); + } + private static KafkaAdminException failedToListTopics(final Throwable ex) { return new KafkaAdminException("Failed to list topics", ex); } @@ -177,6 +186,33 @@ public TopicSettings describe(final String topicName) { .build(); } + /** + * Describes the current configuration of a Kafka topic. + * + * @param topicName the topic name + * @return settings of topic including number of partitions and replicationFactor + */ + public Map getConfig(final String topicName) { + try { + final ConfigResource configResource = new ConfigResource(Type.TOPIC, topicName); + final Map> kafkaTopicMap = + this.adminClient.describeConfigs(List.of(configResource)).values(); + final Config config = kafkaTopicMap.get(configResource).get(this.timeout.toSeconds(), TimeUnit.SECONDS); + return config.entries().stream() + .collect(Collectors.toMap(ConfigEntry::name, ConfigEntry::value)); + } catch (final ExecutionException e) { + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException) e.getCause(); + } + throw failedToRetrieveTopicConfig(topicName, e); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw failedToRetrieveTopicConfig(topicName, e); + } catch (final TimeoutException e) { + throw failedToRetrieveTopicConfig(topicName, e); + } + } + @Override public void close() { this.adminClient.close(); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index e5319771..61825ca5 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -24,11 +24,18 @@ package com.bakdata.kafka; +import static com.bakdata.kafka.KafkaTest.POLL_TIMEOUT; + import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; import com.bakdata.kafka.util.TopologyInformation; import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.clients.producer.ProducerRecord; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.Topology.AutoOffsetReset; import org.apache.kafka.streams.TopologyDescription.Node; import org.assertj.core.api.SoftAssertions; @@ -36,6 +43,7 @@ import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) class ConsumedXTest { @@ -267,15 +275,39 @@ public void buildTopology(final TopologyBuilder builder) { input.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test existing records. TestDriver cannot have existing records + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + TestHelper.run(runner); + KafkaTest.awaitActive(executableApp); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("baz", "qux"))); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.value()).isEqualTo("qux"); + }); + } } } @@ -289,15 +321,39 @@ public void buildTopology(final TopologyBuilder builder) { input.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test existing records. TestDriver cannot have existing records + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + TestHelper.run(runner); + KafkaTest.awaitActive(executableApp); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("baz", "qux"))); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.value()).isEqualTo("qux"); + }); + } } } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java index 5841f1d1..fbacf7ce 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/DoubleApp.java @@ -40,10 +40,11 @@ public SerdeConfig defaultSerializationConfig() { } TestTopology startApp(final StreamsTopicConfig topicConfig) { - return TestHelper.startApp(this, topicConfig); + final ConfiguredStreamsApp configuredApp = TestHelper.configureApp(this, topicConfig); + return TestHelper.startApp(configuredApp); } TestTopology startApp() { - return TestHelper.startApp(this, StreamsTopicConfig.builder().build()); + return this.startApp(StreamsTopicConfig.builder().build()); } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java index 89cba097..3a6c6189 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java @@ -24,14 +24,23 @@ package com.bakdata.kafka; -import static java.util.Collections.emptyMap; +import static com.bakdata.kafka.KafkaTest.POLL_TIMEOUT; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; +import com.bakdata.kafka.util.ImprovedAdminClient; +import com.bakdata.kafka.util.TopicClient; import com.bakdata.kafka.util.TopologyInformation; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.common.utils.Bytes; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.kstream.Materialized.StoreType; @@ -42,7 +51,6 @@ import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.Stores; import org.apache.kafka.streams.state.internals.CachingKeyValueStore; -import org.apache.kafka.streams.state.internals.MeteredTimestampedKeyValueStore; import org.apache.kafka.streams.state.internals.WrappedStateStore; import org.assertj.core.api.InstanceOfAssertFactories; import org.assertj.core.api.SoftAssertions; @@ -50,6 +58,7 @@ import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) class MaterializedXTest { @@ -346,20 +355,44 @@ void shouldDisableLogging() { @Override public void buildTopology(final TopologyBuilder builder) { final KTableX table = builder.table("input", - MaterializedX.>keySerde( - Preconfigured.defaultSerde()).withLoggingDisabled()); + MaterializedX.>as("store") + .withLoggingDisabled()); table.toStream().to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test topic existence. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + }); + try (final ImprovedAdminClient admin = testClient.admin(); + final TopicClient topicClient = admin.getTopicClient()) { + final String appId = new ImprovedStreamsConfig(executableApp.getConfig()).getAppId(); + this.softly.assertThat(topicClient.exists(appId + "-store-changelog")).isFalse(); + } + } } } @@ -369,20 +402,47 @@ void shouldEnableLogging() { @Override public void buildTopology(final TopologyBuilder builder) { final KTableX table = builder.table("input", - MaterializedX.>keySerde( - Preconfigured.defaultSerde()).withLoggingEnabled(emptyMap())); + MaterializedX.>as("store") + .withLoggingEnabled(Map.of(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.1"))); table.toStream().to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input("input") - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test topic config. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + }); + try (final ImprovedAdminClient admin = testClient.admin(); + final TopicClient topicClient = admin.getTopicClient()) { + final String appId = new ImprovedStreamsConfig(executableApp.getConfig()).getAppId(); + final String topicName = appId + "-store-changelog"; + this.softly.assertThat(topicClient.exists(topicName)).isTrue(); + final Map config = topicClient.getConfig(topicName); + this.softly.assertThat(config).containsEntry(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.1"); + } + } } } @@ -406,7 +466,7 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); final TopologyTestDriver testDriver = topology.getTestDriver(); this.softly.assertThat(testDriver.getTimestampedKeyValueStore("store")) - .asInstanceOf(InstanceOfAssertFactories.type(MeteredTimestampedKeyValueStore.class)) + .asInstanceOf(InstanceOfAssertFactories.type(WrappedStateStore.class)) .extracting(WrappedStateStore::wrapped) .isNotInstanceOf(CachingKeyValueStore.class); } @@ -432,9 +492,11 @@ public void buildTopology(final TopologyBuilder builder) { .expectNoMoreRecord(); final TopologyTestDriver testDriver = topology.getTestDriver(); this.softly.assertThat(testDriver.getTimestampedKeyValueStore("store")) - .asInstanceOf(InstanceOfAssertFactories.type(MeteredTimestampedKeyValueStore.class)) + .asInstanceOf(InstanceOfAssertFactories.type(WrappedStateStore.class)) .extracting(WrappedStateStore::wrapped) .isInstanceOf(CachingKeyValueStore.class); } } + + //TODO retention } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java index 2c7b03f8..70ee237d 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ProducedXTest.java @@ -24,15 +24,24 @@ package com.bakdata.kafka; +import static com.bakdata.kafka.KafkaTest.POLL_TIMEOUT; + import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; import com.bakdata.kafka.util.TopologyInformation; +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.TopologyDescription.Node; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) class ProducedXTest { @@ -209,19 +218,47 @@ void shouldUseStreamPartitioner() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - input.to("output", ProducedX.streamPartitioner((topic, key, value, numPartitions) -> 1)); + input.to("output", + ProducedX.streamPartitioner((topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1)); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output", KafkaTestClient.defaultTopicSettings().partitions(2).build()); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of( + new SimpleProducerRecord<>("foo", "bar"), + new SimpleProducerRecord<>("foo", "baz") + )); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(2) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(0); + }) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(1); + }); + } } } @@ -232,19 +269,46 @@ void shouldUseStreamPartitionerModifier() { public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); input.to("output", ProducedX.as("output") - .withStreamPartitioner((topic, key, value, numPartitions) -> 1)); + .withStreamPartitioner((topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1)); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output", KafkaTestClient.defaultTopicSettings().partitions(2).build()); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of( + new SimpleProducerRecord<>("foo", "bar"), + new SimpleProducerRecord<>("foo", "baz") + )); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(2) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(0); + }) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(1); + }); + } } } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java index f12dd343..09727b23 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/RepartitionedXTest.java @@ -24,14 +24,26 @@ package com.bakdata.kafka; +import static com.bakdata.kafka.KafkaTest.POLL_TIMEOUT; + import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; +import com.bakdata.kafka.util.ImprovedAdminClient; +import com.bakdata.kafka.util.TopicClient; +import com.bakdata.kafka.util.TopicSettings; import com.bakdata.kafka.util.TopologyInformation; +import java.util.List; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; import org.assertj.core.api.SoftAssertions; import org.assertj.core.api.junit.jupiter.InjectSoftAssertions; import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) class RepartitionedXTest { @@ -218,21 +230,65 @@ void shouldUseStreamPartitioner() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - final KStreamX repartitioned = - input.repartition(RepartitionedX.streamPartitioner((topic, key, value, numPartitions) -> 1)); + final KStreamX repartitioned = input.repartition(RepartitionedX + .streamPartitioner( + (topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1) + .withNumberOfPartitions(2) + .withName("repartition")); repartitioned.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of( + new SimpleProducerRecord<>("foo", "bar"), + new SimpleProducerRecord<>("foo", "baz") + )); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from(new ImprovedStreamsConfig(executableApp.getConfig()).getAppId() + + "-repartition-repartition", POLL_TIMEOUT)) + .hasSize(2) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(0); + }) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(1); + }); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(2) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + }) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("baz"); + }); + } } } @@ -244,20 +300,63 @@ public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); final KStreamX repartitioned = input.repartition( RepartitionedX.as("repartition") - .withStreamPartitioner((topic, key, value, numPartitions) -> 1)); + .withStreamPartitioner( + (topic, key, value, numPartitions) -> "bar".equals(value) ? 0 : 1) + .withNumberOfPartitions(2)); repartitioned.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of( + new SimpleProducerRecord<>("foo", "bar"), + new SimpleProducerRecord<>("foo", "baz") + )); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from(new ImprovedStreamsConfig(executableApp.getConfig()).getAppId() + + "-repartition-repartition", POLL_TIMEOUT)) + .hasSize(2) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(0); + }) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("baz"); + this.softly.assertThat(outputRecord.partition()).isEqualTo(1); + }); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(2) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + }) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("baz"); + }); + } } } @@ -267,20 +366,45 @@ void shouldUseNumberOfPartitions() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - final KStreamX repartitioned = input.repartition(RepartitionedX.numberOfPartitions(2)); + final KStreamX repartitioned = input.repartition( + RepartitionedX.numberOfPartitions(2).withName("repartition")); repartitioned.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + }); + try (final ImprovedAdminClient admin = testClient.admin(); + final TopicClient topicClient = admin.getTopicClient()) { + final String appId = new ImprovedStreamsConfig(executableApp.getConfig()).getAppId(); + final TopicSettings settings = topicClient.describe(appId + "-repartition-repartition"); + this.softly.assertThat(settings.getPartitions()).isEqualTo(2); + } + } } } @@ -295,16 +419,40 @@ public void buildTopology(final TopologyBuilder builder) { repartitioned.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input() - .at(0L) - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("bar") - .expectNoMoreRecord(); - // TODO test partition. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("bar"); + }); + try (final ImprovedAdminClient admin = testClient.admin(); + final TopicClient topicClient = admin.getTopicClient()) { + final String appId = new ImprovedStreamsConfig(executableApp.getConfig()).getAppId(); + final TopicSettings settings = topicClient.describe(appId + "-repartition-repartition"); + this.softly.assertThat(settings.getPartitions()).isEqualTo(2); + } + } } } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java index 57909517..d0cfd81e 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java @@ -24,14 +24,23 @@ package com.bakdata.kafka; -import static java.util.Collections.emptyMap; +import static com.bakdata.kafka.KafkaTest.POLL_TIMEOUT; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; +import com.bakdata.kafka.util.ImprovedAdminClient; +import com.bakdata.kafka.util.TopicClient; import com.bakdata.kafka.util.TopologyInformation; import java.time.Duration; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.config.TopicConfig; import org.apache.kafka.common.serialization.Serdes; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; import org.apache.kafka.streams.TopologyDescription.Node; import org.apache.kafka.streams.TopologyTestDriver; import org.apache.kafka.streams.kstream.JoinWindows; @@ -43,6 +52,7 @@ import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) class StreamJoinedXTest { @@ -522,22 +532,51 @@ public void buildTopology(final TopologyBuilder builder) { final KStreamX otherInput = builder.stream("other_input"); final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.keySerde(Preconfigured.defaultSerde()) + StreamJoinedX.as("store") .withLoggingDisabled()); joined.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input("other_input") - .add("foo", "baz"); - topology.input("input") - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - // TODO test topic existence. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("other_input", List.of(new SimpleProducerRecord<>("foo", "baz"))); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("barbaz"); + }); + try (final ImprovedAdminClient admin = testClient.admin(); + final TopicClient topicClient = admin.getTopicClient()) { + final String appId = new ImprovedStreamsConfig(executableApp.getConfig()).getAppId(); + this.softly.assertThat(topicClient.listTopics()) + .noneSatisfy(topic -> this.softly.assertThat(topic) + .startsWith(appId) + .endsWith("-store-changelog")); + } + } } } @@ -550,24 +589,54 @@ public void buildTopology(final TopologyBuilder builder) { final KStreamX otherInput = builder.stream("other_input"); final KStreamX joined = stream.join(otherInput, (v1, v2) -> v1 + v2, JoinWindows.ofTimeDifferenceWithNoGrace(Duration.ofMinutes(1L)), - StreamJoinedX.keySerde(Preconfigured.defaultSerde()) - .withLoggingEnabled(emptyMap())); + StreamJoinedX.as("store") + .withLoggingEnabled(Map.of(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.1"))); joined.to("output"); } }; - try (final TestTopology topology = app.startApp()) { - topology.input("other_input") - .add("foo", "baz"); - topology.input("input") - .add("foo", "bar"); - topology.streamOutput() - .expectNextRecord() - .hasKey("foo") - .hasValue("barbaz") - .expectNoMoreRecord(); - // TODO test topic config. TestDriver does not expose it + try (final KafkaContainer kafkaCluster = KafkaTest.newCluster()) { + kafkaCluster.start(); + final KafkaEndpointConfig endpointConfig = KafkaEndpointConfig.builder() + .bootstrapServers(kafkaCluster.getBootstrapServers()) + .build(); + final KafkaTestClient testClient = new KafkaTestClient(endpointConfig); + testClient.createTopic("input"); + testClient.createTopic("output"); + try (final ConfiguredStreamsApp configuredApp = app.configureApp( + TestTopologyFactory.createStreamsTestConfig()); + final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); + final StreamsRunner runner = executableApp.createRunner()) { + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("other_input", List.of(new SimpleProducerRecord<>("foo", "baz"))); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) + .to("input", List.of(new SimpleProducerRecord<>("foo", "bar"))); + TestHelper.run(runner); + KafkaTest.awaitProcessing(executableApp); + this.softly.assertThat(testClient.read() + .with(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .with(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class) + .from("output", POLL_TIMEOUT)) + .hasSize(1) + .anySatisfy(outputRecord -> { + this.softly.assertThat(outputRecord.key()).isEqualTo("foo"); + this.softly.assertThat(outputRecord.value()).isEqualTo("barbaz"); + }); + try (final ImprovedAdminClient admin = testClient.admin(); + final TopicClient topicClient = admin.getTopicClient()) { + final String appId = new ImprovedStreamsConfig(executableApp.getConfig()).getAppId(); + this.softly.assertThat(topicClient.listTopics()) + .filteredOn(topic -> topic.startsWith(appId) && topic.endsWith("-store-changelog")) + .allSatisfy(topic -> { + final Map config = topicClient.getConfig(topic); + this.softly.assertThat(config) + .containsEntry(TopicConfig.MIN_CLEANABLE_DIRTY_RATIO_CONFIG, "0.1"); + }); + } + } } } - - //TODO retention } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java index 35066da8..13c26cd4 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StringApp.java @@ -25,18 +25,11 @@ package com.bakdata.kafka; import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.util.Map; import org.apache.kafka.common.serialization.Serdes.StringSerde; abstract class StringApp implements StreamsApp { - TestTopology startApp(final StreamsTopicConfig topicConfig) { - return TestHelper.startApp(this, topicConfig); - } - - TestTopology startApp() { - return TestHelper.startApp(this, StreamsTopicConfig.builder().build()); - } - @Override public String getUniqueAppId(final StreamsTopicConfig topics) { return "my-app"; @@ -46,4 +39,17 @@ public String getUniqueAppId(final StreamsTopicConfig topics) { public SerdeConfig defaultSerializationConfig() { return new SerdeConfig(StringSerde.class, StringSerde.class); } + + TestTopology startApp(final StreamsTopicConfig topicConfig) { + final ConfiguredStreamsApp configuredApp = TestHelper.configureApp(this, topicConfig); + return TestHelper.startApp(configuredApp); + } + + TestTopology startApp() { + return this.startApp(StreamsTopicConfig.builder().build()); + } + + ConfiguredStreamsApp configureApp(final Map config) { + return TestHelper.configureApp(this, StreamsTopicConfig.builder().build(), config); + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java index c919ddf1..5bd6e43f 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/TestHelper.java @@ -24,17 +24,50 @@ package com.bakdata.kafka; +import static java.util.Collections.emptyMap; + import com.bakdata.fluent_kafka_streams_tests.TestTopology; +import java.lang.Thread.UncaughtExceptionHandler; +import java.util.Map; +import lombok.Getter; import lombok.experimental.UtilityClass; @UtilityClass -class TestHelper { - static TestTopology startApp(final StreamsApp app, final StreamsTopicConfig topicConfig) { - final ConfiguredStreamsApp configuredStreamsApp = - new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig)); +public class TestHelper { + public static Thread run(final StreamsRunner runner) { + // run in Thread because the application blocks indefinitely + final Thread thread = new Thread(runner); + final UncaughtExceptionHandler handler = new CapturingUncaughtExceptionHandler(); + thread.setUncaughtExceptionHandler(handler); + thread.start(); + return thread; + } + + static TestTopology startApp(final ConfiguredStreamsApp app) { final TestTopology topology = - TestTopologyFactory.withoutSchemaRegistry().createTopology(configuredStreamsApp); + TestTopologyFactory.withoutSchemaRegistry().createTopology(app); topology.start(); return topology; } + + static ConfiguredStreamsApp configureApp(final StreamsApp app, + final StreamsTopicConfig topicConfig) { + return configureApp(app, topicConfig, emptyMap()); + } + + static ConfiguredStreamsApp configureApp(final StreamsApp app, + final StreamsTopicConfig topicConfig, + final Map config) { + return new ConfiguredStreamsApp<>(app, new AppConfiguration<>(topicConfig, config)); + } + + @Getter + public static class CapturingUncaughtExceptionHandler implements UncaughtExceptionHandler { + private Throwable lastException; + + @Override + public void uncaughtException(final Thread t, final Throwable e) { + this.lastException = e; + } + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java index 016804b6..807e179c 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java @@ -43,6 +43,7 @@ import com.bakdata.kafka.StreamsCleanUpRunner; import com.bakdata.kafka.StreamsRunner; import com.bakdata.kafka.StreamsTopicConfig; +import com.bakdata.kafka.TestHelper; import com.bakdata.kafka.TestRecord; import com.bakdata.kafka.test_applications.ComplexTopologyApplication; import com.bakdata.kafka.test_applications.MirrorKeyWithAvro; @@ -110,6 +111,14 @@ ConfiguredStreamsApp configureApp(final StreamsApp app, final Stream return StreamsRunnerTest.configureApp(app, topics, this.stateDir); } + public static void run(final ExecutableStreamsApp app) { + try (final StreamsRunner runner = app.createRunner()) { + TestHelper.run(runner); + // Wait until stream application has consumed all data + awaitProcessing(app); + } + } + @Test void shouldDeleteTopic() { try (final ConfiguredStreamsApp app = this.createWordCountApplication(); @@ -132,11 +141,11 @@ void shouldDeleteTopic() { new KeyValue<>("blub", 2L) ); - this.run(executableApp); + run(executableApp); this.assertContent(app.getTopics().getOutputTopic(), expectedValues, "WordCount contains all elements after first run"); - this.awaitClosed(executableApp); + awaitClosed(executableApp); clean(executableApp); try (final ImprovedAdminClient admin = testClient.admin(); @@ -170,7 +179,7 @@ void shouldDeleteConsumerGroup() { new KeyValue<>("blub", 2L) ); - this.run(executableApp); + run(executableApp); this.assertContent(app.getTopics().getOutputTopic(), expectedValues, "WordCount contains all elements after first run"); @@ -181,7 +190,7 @@ void shouldDeleteConsumerGroup() { .isTrue(); } - this.awaitClosed(executableApp); + awaitClosed(executableApp); clean(executableApp); try (final ImprovedAdminClient adminClient = testClient.admin(); @@ -215,7 +224,7 @@ void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { new KeyValue<>("blub", 2L) ); - this.run(executableApp); + run(executableApp); this.assertContent(app.getTopics().getOutputTopic(), expectedValues, "WordCount contains all elements after first run"); @@ -226,7 +235,7 @@ void shouldNotThrowAnErrorIfConsumerGroupDoesNotExist() { .isTrue(); } - this.awaitClosed(executableApp); + awaitClosed(executableApp); try (final ImprovedAdminClient adminClient = testClient.admin(); final ConsumerGroupClient consumerGroupClient = adminClient.getConsumerGroupClient()) { @@ -254,7 +263,7 @@ void shouldDeleteInternalTopics() { new SimpleProducerRecord<>("key 1", testRecord) )); - this.run(executableApp); + run(executableApp); final List inputTopics = app.getTopics().getInputTopics(); final String uniqueAppId = app.getUniqueAppId(); @@ -274,7 +283,7 @@ void shouldDeleteInternalTopics() { this.softly.assertThat(topicClient.exists(manualTopic)).isTrue(); } - this.awaitClosed(executableApp); + awaitClosed(executableApp); reset(executableApp); try (final ImprovedAdminClient admin = testClient.admin(); @@ -304,7 +313,7 @@ void shouldDeleteIntermediateTopics() { new SimpleProducerRecord<>("key 1", testRecord) )); - this.run(executableApp); + run(executableApp); final List inputTopics = app.getTopics().getInputTopics(); final String manualTopic = ComplexTopologyApplication.THROUGH_TOPIC; @@ -317,7 +326,7 @@ void shouldDeleteIntermediateTopics() { this.softly.assertThat(topicClient.exists(manualTopic)).isTrue(); } - this.awaitClosed(executableApp); + awaitClosed(executableApp); clean(executableApp); try (final ImprovedAdminClient admin = testClient.admin(); @@ -352,14 +361,14 @@ void shouldDeleteState() { new KeyValue<>("blub", 2L) ); - this.run(executableApp); + run(executableApp); this.assertContent(app.getTopics().getOutputTopic(), expectedValues, "All entries are once in the input topic after the 1st run"); - this.awaitClosed(executableApp); + awaitClosed(executableApp); reset(executableApp); - this.run(executableApp); + run(executableApp); final List> entriesTwice = expectedValues.stream() .flatMap(entry -> Stream.of(entry, entry)) .collect(Collectors.toList()); @@ -384,16 +393,16 @@ void shouldReprocessAlreadySeenRecords() { new SimpleProducerRecord<>(null, "c") )); - this.run(executableApp); + run(executableApp); this.assertSize(app.getTopics().getOutputTopic(), 3); - this.run(executableApp); + run(executableApp); this.assertSize(app.getTopics().getOutputTopic(), 3); // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(executableApp); + awaitClosed(executableApp); reset(executableApp); - this.run(executableApp); + run(executableApp); this.assertSize(app.getTopics().getOutputTopic(), 6); } } @@ -416,10 +425,10 @@ void shouldDeleteValueSchema() new SimpleProducerRecord<>(null, testRecord) )); - this.run(executableApp); + run(executableApp); // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(executableApp); + awaitClosed(executableApp); final String outputTopic = app.getTopics().getOutputTopic(); this.softly.assertThat(client.getAllSubjects()) .contains(outputTopic + "-value", inputTopic + "-value"); @@ -447,10 +456,10 @@ void shouldDeleteKeySchema() new SimpleProducerRecord<>(testRecord, "val") )); - this.run(executableApp); + run(executableApp); // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(executableApp); + awaitClosed(executableApp); final String outputTopic = app.getTopics().getOutputTopic(); this.softly.assertThat(client.getAllSubjects()) .contains(outputTopic + "-key", inputTopic + "-key"); @@ -478,10 +487,10 @@ void shouldDeleteSchemaOfInternalTopics() new SimpleProducerRecord<>("key 1", testRecord) )); - this.run(executableApp); + run(executableApp); // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(executableApp); + awaitClosed(executableApp); final String inputSubject = inputTopic + "-value"; final String uniqueAppId = app.getUniqueAppId(); final String internalSubject = @@ -499,39 +508,6 @@ void shouldDeleteSchemaOfInternalTopics() } } - @Test - void shouldDeleteSchemaOfIntermediateTopics() - throws IOException, RestClientException { - try (final ConfiguredStreamsApp app = this.createComplexApplication(); - final ExecutableStreamsApp executableApp = app.withEndpoint(this.createEndpoint()); - final SchemaRegistryClient client = this.getSchemaRegistryClient()) { - final TestRecord testRecord = TestRecord.newBuilder().setContent("key 1").build(); - final String inputTopic = app.getTopics().getInputTopics().get(0); - final KafkaTestClient testClient = this.newTestClient(); - testClient.createTopic(app.getTopics().getOutputTopic()); - testClient.send() - .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()) - .to(app.getTopics().getInputTopics().get(0), List.of( - new SimpleProducerRecord<>("key 1", testRecord) - )); - - this.run(executableApp); - - // Wait until all stream applications are completely stopped before triggering cleanup - this.awaitClosed(executableApp); - final String inputSubject = inputTopic + "-value"; - final String manualSubject = ComplexTopologyApplication.THROUGH_TOPIC + "-value"; - this.softly.assertThat(client.getAllSubjects()) - .contains(inputSubject, manualSubject); - clean(executableApp); - - this.softly.assertThat(client.getAllSubjects()) - .doesNotContain(manualSubject) - .contains(inputSubject); - } - } - @Test void shouldCallCleanupHookForInternalTopics() { try (final ConfiguredStreamsApp app = this.createComplexCleanUpHookApplication(); @@ -570,6 +546,39 @@ void shouldNotThrowExceptionOnMissingInputTopic() { } } + @Test + void shouldDeleteSchemaOfIntermediateTopics() + throws IOException, RestClientException { + try (final ConfiguredStreamsApp app = this.createComplexApplication(); + final ExecutableStreamsApp executableApp = app.withEndpoint(this.createEndpoint()); + final SchemaRegistryClient client = this.getSchemaRegistryClient()) { + final TestRecord testRecord = TestRecord.newBuilder().setContent("key 1").build(); + final String inputTopic = app.getTopics().getInputTopics().get(0); + final KafkaTestClient testClient = this.newTestClient(); + testClient.createTopic(app.getTopics().getOutputTopic()); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()) + .to(app.getTopics().getInputTopics().get(0), List.of( + new SimpleProducerRecord<>("key 1", testRecord) + )); + + run(executableApp); + + // Wait until all stream applications are completely stopped before triggering cleanup + awaitClosed(executableApp); + final String inputSubject = inputTopic + "-value"; + final String manualSubject = ComplexTopologyApplication.THROUGH_TOPIC + "-value"; + this.softly.assertThat(client.getAllSubjects()) + .contains(inputSubject, manualSubject); + clean(executableApp); + + this.softly.assertThat(client.getAllSubjects()) + .doesNotContain(manualSubject) + .contains(inputSubject); + } + } + @Test void shouldThrowExceptionOnResetterError() { try (final ConfiguredStreamsApp app = this.createMirrorKeyApplication(); @@ -577,9 +586,9 @@ void shouldThrowExceptionOnResetterError() { final StreamsRunner runner = executableApp.createRunner()) { final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(app.getTopics().getInputTopics().get(0)); - StreamsRunnerTest.run(runner); + TestHelper.run(runner); // Wait until stream application has consumed all data - this.awaitActive(executableApp); + awaitActive(executableApp); // should throw exception because consumer group is still active this.softly.assertThatThrownBy(() -> reset(executableApp)) .isInstanceOf(CleanUpException.class) @@ -608,16 +617,16 @@ void shouldReprocessAlreadySeenRecordsWithPattern() { new SimpleProducerRecord<>(null, "c") )); - this.run(executableApp); + run(executableApp); this.assertSize(app.getTopics().getOutputTopic(), 3); - this.run(executableApp); + run(executableApp); this.assertSize(app.getTopics().getOutputTopic(), 3); // Wait until all streams application are completely stopped before triggering cleanup - this.awaitClosed(executableApp); + awaitClosed(executableApp); reset(executableApp); - this.run(executableApp); + run(executableApp); this.assertSize(app.getTopics().getOutputTopic(), 6); } } @@ -650,14 +659,6 @@ private ConfiguredStreamsApp createMirrorKeyApplication() { .build()); } - private void run(final ExecutableStreamsApp app) { - try (final StreamsRunner runner = app.createRunner()) { - StreamsRunnerTest.run(runner); - // Wait until stream application has consumed all data - this.awaitProcessing(app); - } - } - private ConfiguredStreamsApp createComplexApplication() { this.newTestClient().createTopic(ComplexTopologyApplication.THROUGH_TOPIC); return this.configureApp(new ComplexTopologyApplication(), StreamsTopicConfig.builder() diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java index 8c6b919d..6f995bbe 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java @@ -41,15 +41,15 @@ import com.bakdata.kafka.StreamsExecutionOptions; import com.bakdata.kafka.StreamsRunner; import com.bakdata.kafka.StreamsTopicConfig; +import com.bakdata.kafka.TestHelper; +import com.bakdata.kafka.TestHelper.CapturingUncaughtExceptionHandler; import com.bakdata.kafka.TopologyBuilder; import com.bakdata.kafka.test_applications.LabeledInputTopics; import com.bakdata.kafka.test_applications.Mirror; -import java.lang.Thread.UncaughtExceptionHandler; import java.nio.file.Path; import java.time.Duration; import java.util.List; import java.util.Map; -import lombok.Getter; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.producer.ProducerConfig; import org.apache.kafka.common.serialization.Serdes.StringSerde; @@ -85,15 +85,6 @@ class StreamsRunnerTest extends KafkaTest { @TempDir private Path stateDir; - static Thread run(final StreamsRunner runner) { - // run in Thread because the application blocks indefinitely - final Thread thread = new Thread(runner); - final UncaughtExceptionHandler handler = new CapturingUncaughtExceptionHandler(); - thread.setUncaughtExceptionHandler(handler); - thread.start(); - return thread; - } - static ConfiguredStreamsApp configureApp(final StreamsApp app, final StreamsTopicConfig topics, final Path stateDir) { final AppConfiguration configuration = @@ -118,7 +109,7 @@ void shouldRunApp() { final String outputTopic = app.getTopics().getOutputTopic(); final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(outputTopic); - run(runner); + TestHelper.run(runner); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) @@ -144,7 +135,7 @@ void shouldUseMultipleLabeledInputTopics() { testClient.createTopic(inputTopic1); testClient.createTopic(inputTopic2); testClient.createTopic(outputTopic); - run(runner); + TestHelper.run(runner); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) @@ -170,7 +161,7 @@ void shouldThrowOnMissingInputTopic() { .stateListener(() -> this.stateListener) .uncaughtExceptionHandler(() -> this.uncaughtExceptionHandler) .build())) { - final Thread thread = run(runner); + final Thread thread = TestHelper.run(runner); final CapturingUncaughtExceptionHandler handler = (CapturingUncaughtExceptionHandler) thread.getUncaughtExceptionHandler(); awaitThreadIsDead(thread); @@ -193,7 +184,7 @@ void shouldCloseOnMapError() { final String outputTopic = app.getTopics().getOutputTopic(); final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(outputTopic); - final Thread thread = run(runner); + final Thread thread = TestHelper.run(runner); final CapturingUncaughtExceptionHandler handler = (CapturingUncaughtExceptionHandler) thread.getUncaughtExceptionHandler(); testClient.send() @@ -229,16 +220,6 @@ private ConfiguredStreamsApp createErrorApplication() { .build()); } - @Getter - private static class CapturingUncaughtExceptionHandler implements UncaughtExceptionHandler { - private Throwable lastException; - - @Override - public void uncaughtException(final Thread t, final Throwable e) { - this.lastException = e; - } - } - private static class ErrorApplication implements StreamsApp { @Override diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/util/TopicClientTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/util/TopicClientTest.java index 9b3845dd..26021ae9 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/util/TopicClientTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/util/TopicClientTest.java @@ -32,6 +32,7 @@ import java.time.Duration; import java.util.Map; import org.apache.kafka.clients.admin.AdminClientConfig; +import org.apache.kafka.common.config.TopicConfig; import org.junit.jupiter.api.Test; class TopicClientTest extends KafkaTest { @@ -94,6 +95,20 @@ void shouldCreateTopic() { } } + @Test + void shouldGetTopicConfig() { + try (final TopicClient client = this.createClient()) { + final Map config = Map.of( + TopicConfig.CLEANUP_POLICY_CONFIG, + TopicConfig.CLEANUP_POLICY_COMPACT + "," + TopicConfig.CLEANUP_POLICY_DELETE + ); + client.createTopic("foo", defaultTopicSettings().build(), config); + assertThat(client.getConfig("foo")) + .containsEntry(TopicConfig.CLEANUP_POLICY_CONFIG, + TopicConfig.CLEANUP_POLICY_COMPACT + "," + TopicConfig.CLEANUP_POLICY_DELETE); + } + } + private TopicClient createClient() { final String brokerList = this.getBootstrapServers(); final Map config = Map.of(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList); diff --git a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java index 20a05f59..d68983df 100644 --- a/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java +++ b/streams-bootstrap-core/src/testFixtures/java/com/bakdata/kafka/KafkaTest.java @@ -45,6 +45,28 @@ public static KafkaContainer newCluster() { .withTag("3.8.1")); } + protected static void awaitProcessing(final ExecutableStreamsApp app) { + awaitActive(app); + final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(app); + await() + .alias("Consumer group has finished processing") + .until(verifier::hasFinishedProcessing); + } + + protected static void awaitActive(final ExecutableStreamsApp app) { + final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(app); + await() + .alias("Consumer group is active") + .until(verifier::isActive); + } + + protected static void awaitClosed(final ExecutableStreamsApp app) { + final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(app); + await() + .alias("Consumer group is closed") + .until(verifier::isClosed); + } + private static ConditionFactory await() { return Awaitility.await() .pollInterval(Duration.ofSeconds(2L)) @@ -80,26 +102,4 @@ protected SchemaRegistryClient getSchemaRegistryClient() { return this.testTopologyFactory.getSchemaRegistryClient(); } - protected void awaitProcessing(final ExecutableStreamsApp app) { - this.awaitActive(app); - final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(app); - await() - .alias("Consumer group has finished processing") - .until(verifier::hasFinishedProcessing); - } - - protected void awaitActive(final ExecutableStreamsApp app) { - final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(app); - await() - .alias("Consumer group is active") - .until(verifier::isActive); - } - - protected void awaitClosed(final ExecutableStreamsApp app) { - final ConsumerGroupVerifier verifier = ConsumerGroupVerifier.verify(app); - await() - .alias("Consumer group is closed") - .until(verifier::isClosed); - } - } From 37f3344321614090d588658b6d1eeccee456d442 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 20:21:11 +0100 Subject: [PATCH 66/72] Add tests --- .../com/bakdata/kafka/MaterializedXTest.java | 59 +++++++++++++++++-- .../com/bakdata/kafka/StreamJoinedXTest.java | 10 ++-- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java index 3a6c6189..26997a66 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/MaterializedXTest.java @@ -25,12 +25,14 @@ package com.bakdata.kafka; import static com.bakdata.kafka.KafkaTest.POLL_TIMEOUT; +import static java.util.Collections.emptyMap; import com.bakdata.fluent_kafka_streams_tests.TestTopology; import com.bakdata.kafka.SenderBuilder.SimpleProducerRecord; import com.bakdata.kafka.util.ImprovedAdminClient; import com.bakdata.kafka.util.TopicClient; import com.bakdata.kafka.util.TopologyInformation; +import java.nio.file.Path; import java.time.Duration; import java.util.List; import java.util.Map; @@ -50,6 +52,7 @@ import org.apache.kafka.streams.processor.StateStore; import org.apache.kafka.streams.state.KeyValueStore; import org.apache.kafka.streams.state.Stores; +import org.apache.kafka.streams.state.WindowStore; import org.apache.kafka.streams.state.internals.CachingKeyValueStore; import org.apache.kafka.streams.state.internals.WrappedStateStore; import org.assertj.core.api.InstanceOfAssertFactories; @@ -58,6 +61,7 @@ import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) @@ -350,7 +354,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldDisableLogging() { + void shouldDisableLogging(@TempDir final Path stateDir) { final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -369,7 +373,7 @@ public void buildTopology(final TopologyBuilder builder) { testClient.createTopic("input"); testClient.createTopic("output"); try (final ConfiguredStreamsApp configuredApp = app.configureApp( - TestTopologyFactory.createStreamsTestConfig()); + TestTopologyFactory.createStreamsTestConfig(stateDir)); final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); final StreamsRunner runner = executableApp.createRunner()) { testClient.send() @@ -397,7 +401,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldEnableLogging() { + void shouldEnableLogging(@TempDir final Path stateDir) { final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -416,7 +420,7 @@ public void buildTopology(final TopologyBuilder builder) { testClient.createTopic("input"); testClient.createTopic("output"); try (final ConfiguredStreamsApp configuredApp = app.configureApp( - TestTopologyFactory.createStreamsTestConfig()); + TestTopologyFactory.createStreamsTestConfig(stateDir)); final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); final StreamsRunner runner = executableApp.createRunner()) { testClient.send() @@ -498,5 +502,50 @@ public void buildTopology(final TopologyBuilder builder) { } } - //TODO retention + @Test + void shouldUseRetention() { + final StringApp app = new StringApp() { + @Override + public void buildTopology(final TopologyBuilder builder) { + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMinutes(1L))); + final KTableX, Long> counted = windowed.count( + MaterializedX.>as("store") + .withRetention(Duration.ofMinutes(1L))); + counted.toStream((k, v) -> k.key() + ":" + k.window().startTime().toEpochMilli()).to("output"); + } + }; + try (final TestTopology topology = app.startApp()) { + topology.input("input") + .add("foo", "bar") + .at(Duration.ofMinutes(2L).toMillis()) + .add("foo", "bar"); + topology.streamOutput() + .withValueSerde(Serdes.Long()) + .expectNextRecord() + .hasKey("foo:0") + .hasValue(1L) + .expectNextRecord() + .expectNoMoreRecord(); + } + } + + @Test + void shouldThrowIfRetentionIsTooShort() { + final TopologyBuilder builder = new TopologyBuilder(StreamsTopicConfig.builder().build(), emptyMap()); + final KStreamX input = builder.stream("input"); + final KGroupedStreamX grouped = input.groupByKey(); + final TimeWindowedKStreamX windowed = + grouped.windowedBy(TimeWindows.ofSizeWithNoGrace(Duration.ofMinutes(1L))); + final MaterializedX> materialized = + MaterializedX.>as("store") + .withRetention(Duration.ofSeconds(1L)); + this.softly.assertThatThrownBy(() -> windowed.count(materialized)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "The retention period of the window store store must be no smaller than its window size plus " + + "the grace period"); + } } diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java index d0cfd81e..2b6d75f6 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/StreamJoinedXTest.java @@ -31,6 +31,7 @@ import com.bakdata.kafka.util.ImprovedAdminClient; import com.bakdata.kafka.util.TopicClient; import com.bakdata.kafka.util.TopologyInformation; +import java.nio.file.Path; import java.time.Duration; import java.util.List; import java.util.Map; @@ -52,6 +53,7 @@ import org.assertj.core.api.junit.jupiter.SoftAssertionsExtension; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; import org.testcontainers.kafka.KafkaContainer; @ExtendWith(SoftAssertionsExtension.class) @@ -524,7 +526,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldDisableLogging() { + void shouldDisableLogging(@TempDir final Path stateDir) { final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -546,7 +548,7 @@ public void buildTopology(final TopologyBuilder builder) { testClient.createTopic("input"); testClient.createTopic("output"); try (final ConfiguredStreamsApp configuredApp = app.configureApp( - TestTopologyFactory.createStreamsTestConfig()); + TestTopologyFactory.createStreamsTestConfig(stateDir)); final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); final StreamsRunner runner = executableApp.createRunner()) { testClient.send() @@ -581,7 +583,7 @@ public void buildTopology(final TopologyBuilder builder) { } @Test - void shouldEnableLogging() { + void shouldEnableLogging(@TempDir final Path stateDir) { final StringApp app = new StringApp() { @Override public void buildTopology(final TopologyBuilder builder) { @@ -603,7 +605,7 @@ public void buildTopology(final TopologyBuilder builder) { testClient.createTopic("input"); testClient.createTopic("output"); try (final ConfiguredStreamsApp configuredApp = app.configureApp( - TestTopologyFactory.createStreamsTestConfig()); + TestTopologyFactory.createStreamsTestConfig(stateDir)); final ExecutableStreamsApp executableApp = configuredApp.withEndpoint(endpointConfig); final StreamsRunner runner = executableApp.createRunner()) { testClient.send() From e8e71aec3453d0e9aa6a632242e86746d313260b Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 20:46:38 +0100 Subject: [PATCH 67/72] Clean up --- .../src/test/java/com/bakdata/kafka/BranchedXTest.java | 2 +- .../src/test/java/com/bakdata/kafka/ConsumedXTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java index c6d407bc..f8c8b6d0 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/BranchedXTest.java @@ -86,7 +86,7 @@ void shouldUseConsumer() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input"); - final Map> branches = input.split() + input.split() .defaultBranch(BranchedX.withConsumer(KStreamX::toOutputTopic)); } }; diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java index 61825ca5..ed5d0353 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/ConsumedXTest.java @@ -221,7 +221,7 @@ void shouldUseTimestampExtractor() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", - ConsumedX.with((record, partitionTime) -> 1L)); + ConsumedX.with((consumerRecord, partitionTime) -> 1L)); input.to("output"); } }; @@ -246,7 +246,7 @@ void shouldUseTimestampExtractorModifier() { @Override public void buildTopology(final TopologyBuilder builder) { final KStreamX input = builder.stream("input", ConsumedX.as("stream") - .withTimestampExtractor((record, partitionTime) -> 1L)); + .withTimestampExtractor((consumerRecord, partitionTime) -> 1L)); input.to("output"); } }; From 682fc4baad4910993c3b1f988ec5bfc6b5bed3c9 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 21:31:09 +0100 Subject: [PATCH 68/72] Update --- .../src/main/java/com/bakdata/kafka/Preconfigured.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/Preconfigured.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/Preconfigured.java index ee9bc3cb..69697c7f 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/Preconfigured.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/Preconfigured.java @@ -1,7 +1,7 @@ /* * MIT License * - * Copyright (c) 2024 bakdata + * Copyright (c) 2025 bakdata * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; import lombok.AccessLevel; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -116,10 +117,12 @@ public static , T> Preconfigured create(final S seria } private static , T> ConfigurableSerde configurable(final S serde) { + Objects.requireNonNull(serde, "Use Preconfigured#defaultSerde instead"); return new ConfigurableSerde<>(serde); } private static , T> ConfigurableSerializer configurable(final S serializer) { + Objects.requireNonNull(serializer, "Use Preconfigured#defaultSerializer instead"); return new ConfigurableSerializer<>(serializer); } From 848bec065ae150f4e154d0daf6c691ec8ca768d5 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 21:31:33 +0100 Subject: [PATCH 69/72] Remove refresh dependencies --- .github/workflows/build-and-publish.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-and-publish.yaml b/.github/workflows/build-and-publish.yaml index 5eab07c4..663472f3 100644 --- a/.github/workflows/build-and-publish.yaml +++ b/.github/workflows/build-and-publish.yaml @@ -12,7 +12,6 @@ jobs: uses: bakdata/ci-templates/.github/workflows/java-gradle-library.yaml@1.62.0 with: java-version: 17 - gradle-refresh-dependencies: true secrets: sonar-token: ${{ secrets.SONARCLOUD_TOKEN }} sonar-organization: ${{ secrets.SONARCLOUD_ORGANIZATION }} From dc85aa32798e981113d3c3394fd689e68680542e Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Wed, 19 Feb 2025 21:48:05 +0100 Subject: [PATCH 70/72] Clean up --- .../kafka/util/TopologyInformation.java | 108 +++++++++++------- .../integration/StreamsCleanUpRunnerTest.java | 66 +++++------ .../kafka/integration/StreamsRunnerTest.java | 10 +- 3 files changed, 102 insertions(+), 82 deletions(-) diff --git a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java index 1d9fd5d4..f69d7a1c 100644 --- a/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java +++ b/streams-bootstrap-core/src/main/java/com/bakdata/kafka/util/TopologyInformation.java @@ -87,46 +87,12 @@ private static List getNodes(final TopologyDescription description) { .collect(Collectors.toList()); } - private static Stream getAllSubscriptions(final Collection nodes) { - return getAllSources(nodes) - .map(TopologyInformation::toSubscription); - } - - private static Stream getAllSources(final Collection nodes) { - return nodes.stream() - .filter(Source.class::isInstance) - .map(Source.class::cast); - } - private static TopicSubscription toSubscription(final Source source) { final Set topicSet = source.topicSet(); return topicSet == null ? new PatternTopicSubscription(source.topicPattern()) : new DirectTopicSubscription(topicSet); } - private static Stream getAllTopics(final Collection nodes) { - return getAllSinks(nodes) - .map(Sink::topic); - } - - private static Stream getAllSinks(final Collection nodes) { - return nodes.stream() - .filter(Sink.class::isInstance) - .map(Sink.class::cast); - } - - private static Stream getAllStores(final Collection nodes) { - return getAllProcessors(nodes) - .flatMap(processor -> processor.stores().stream()) - .distinct(); - } - - private static Stream getAllProcessors(final Collection nodes) { - return nodes.stream() - .filter(Processor.class::isInstance) - .map(Processor.class::cast); - } - private static Stream createPseudoTopics(final String topic) { if (isSubscriptionRegistrationTopic(topic)) { return PSEUDO_TOPIC_SUFFIXES.stream().map(suffix -> String.format("%s%s", topic, suffix)); @@ -166,7 +132,7 @@ public List getInternalTopics() { * @return list of external sink topics */ public List getExternalSinkTopics() { - return getAllTopics(this.nodes) + return this.getAllTopics() .filter(this::isExternalTopic) .collect(Collectors.toList()); } @@ -179,7 +145,7 @@ public List getExternalSinkTopics() { */ public List getExternalSourceTopics(final Collection allTopics) { final List sinks = this.getExternalSinkTopics(); - return getAllSubscriptions(this.nodes) + return this.getAllSubscriptions() .map(subscription -> subscription.resolveTopics(allTopics)) .flatMap(Collection::stream) .filter(this::isExternalTopic) @@ -196,7 +162,7 @@ public List getExternalSourceTopics(final Collection allTopics) */ public List getIntermediateTopics(final Collection allTopics) { final List sinks = this.getExternalSinkTopics(); - return getAllSubscriptions(this.nodes) + return this.getAllSubscriptions() .map(subscription -> subscription.resolveTopics(allTopics)) .flatMap(Collection::stream) .filter(this::isExternalTopic) @@ -204,26 +170,80 @@ public List getIntermediateTopics(final Collection allTopics) { .collect(Collectors.toList()); } + /** + * Retrieve all stores associated with this topology + * + * @return list of stores + */ public List getStores() { - return getAllStores(this.nodes) + return this.getAllStores() .collect(Collectors.toList()); } + /** + * Retrieve all processors associated with this topology + * + * @return list of processors + */ public List getProcessors() { - return getAllProcessors(this.nodes) + return this.getAllProcessors() .collect(Collectors.toList()); } + /** + * Retrieve all sources associated with this topology + * + * @return list of sources + */ public List getSources() { - return getAllSources(this.nodes) + return this.getAllSources() .collect(Collectors.toList()); } + /** + * Retrieve all sinks associated with this topology + * + * @return list of sinks + */ public List getSinks() { - return getAllSinks(this.nodes) + return this.getAllSinks() .collect(Collectors.toList()); } + private Stream getAllSubscriptions() { + return this.getAllSources() + .map(TopologyInformation::toSubscription); + } + + private Stream getAllSources() { + return this.nodes.stream() + .filter(Source.class::isInstance) + .map(Source.class::cast); + } + + private Stream getAllTopics() { + return this.getAllSinks() + .map(Sink::topic); + } + + private Stream getAllSinks() { + return this.nodes.stream() + .filter(Sink.class::isInstance) + .map(Sink.class::cast); + } + + private Stream getAllStores() { + return this.getAllProcessors() + .flatMap(processor -> processor.stores().stream()) + .distinct(); + } + + private Stream getAllProcessors() { + return this.nodes.stream() + .filter(Processor.class::isInstance) + .map(Processor.class::cast); + } + private boolean isInternalTopic(final String topic) { if (topic.startsWith("KSTREAM-") || topic.startsWith("KTABLE-")) { return true; @@ -247,19 +267,19 @@ private boolean isExternalTopic(final String topic) { } private Stream getInternalSinks() { - return getAllTopics(this.nodes) + return this.getAllTopics() .filter(this::isInternalTopic) .flatMap(topic -> Seq.of(topic).concat(createPseudoTopics(topic))) .map(topic -> String.format("%s-%s", this.streamsId, topic)); } private Stream getChangelogTopics() { - return getAllStores(this.nodes) + return this.getAllStores() .map(store -> String.format("%s-%s%s", this.streamsId, store, CHANGELOG_SUFFIX)); } private Stream getRepartitionTopics() { - return getAllProcessors(this.nodes) + return this.getAllProcessors() // internal repartitioning creates one processor that ends with "-repartition-filter", // one sink node, and one source node .filter(processor -> processor.name().endsWith(REPARTITION_SUFFIX + FILTER_SUFFIX)) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java index 807e179c..08348646 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsCleanUpRunnerTest.java @@ -508,6 +508,39 @@ void shouldDeleteSchemaOfInternalTopics() } } + @Test + void shouldDeleteSchemaOfIntermediateTopics() + throws IOException, RestClientException { + try (final ConfiguredStreamsApp app = this.createComplexApplication(); + final ExecutableStreamsApp executableApp = app.withEndpoint(this.createEndpoint()); + final SchemaRegistryClient client = this.getSchemaRegistryClient()) { + final TestRecord testRecord = TestRecord.newBuilder().setContent("key 1").build(); + final String inputTopic = app.getTopics().getInputTopics().get(0); + final KafkaTestClient testClient = this.newTestClient(); + testClient.createTopic(app.getTopics().getOutputTopic()); + testClient.send() + .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) + .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()) + .to(app.getTopics().getInputTopics().get(0), List.of( + new SimpleProducerRecord<>("key 1", testRecord) + )); + + run(executableApp); + + // Wait until all stream applications are completely stopped before triggering cleanup + awaitClosed(executableApp); + final String inputSubject = inputTopic + "-value"; + final String manualSubject = ComplexTopologyApplication.THROUGH_TOPIC + "-value"; + this.softly.assertThat(client.getAllSubjects()) + .contains(inputSubject, manualSubject); + clean(executableApp); + + this.softly.assertThat(client.getAllSubjects()) + .doesNotContain(manualSubject) + .contains(inputSubject); + } + } + @Test void shouldCallCleanupHookForInternalTopics() { try (final ConfiguredStreamsApp app = this.createComplexCleanUpHookApplication(); @@ -546,39 +579,6 @@ void shouldNotThrowExceptionOnMissingInputTopic() { } } - @Test - void shouldDeleteSchemaOfIntermediateTopics() - throws IOException, RestClientException { - try (final ConfiguredStreamsApp app = this.createComplexApplication(); - final ExecutableStreamsApp executableApp = app.withEndpoint(this.createEndpoint()); - final SchemaRegistryClient client = this.getSchemaRegistryClient()) { - final TestRecord testRecord = TestRecord.newBuilder().setContent("key 1").build(); - final String inputTopic = app.getTopics().getInputTopics().get(0); - final KafkaTestClient testClient = this.newTestClient(); - testClient.createTopic(app.getTopics().getOutputTopic()); - testClient.send() - .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()) - .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()) - .to(app.getTopics().getInputTopics().get(0), List.of( - new SimpleProducerRecord<>("key 1", testRecord) - )); - - run(executableApp); - - // Wait until all stream applications are completely stopped before triggering cleanup - awaitClosed(executableApp); - final String inputSubject = inputTopic + "-value"; - final String manualSubject = ComplexTopologyApplication.THROUGH_TOPIC + "-value"; - this.softly.assertThat(client.getAllSubjects()) - .contains(inputSubject, manualSubject); - clean(executableApp); - - this.softly.assertThat(client.getAllSubjects()) - .doesNotContain(manualSubject) - .contains(inputSubject); - } - } - @Test void shouldThrowExceptionOnResetterError() { try (final ConfiguredStreamsApp app = this.createMirrorKeyApplication(); diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java index 6f995bbe..2a5cd70a 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/integration/StreamsRunnerTest.java @@ -24,6 +24,7 @@ package com.bakdata.kafka.integration; +import static com.bakdata.kafka.TestHelper.run; import static com.bakdata.kafka.TestTopologyFactory.createStreamsTestConfig; import static org.awaitility.Awaitility.await; import static org.mockito.ArgumentMatchers.any; @@ -41,7 +42,6 @@ import com.bakdata.kafka.StreamsExecutionOptions; import com.bakdata.kafka.StreamsRunner; import com.bakdata.kafka.StreamsTopicConfig; -import com.bakdata.kafka.TestHelper; import com.bakdata.kafka.TestHelper.CapturingUncaughtExceptionHandler; import com.bakdata.kafka.TopologyBuilder; import com.bakdata.kafka.test_applications.LabeledInputTopics; @@ -109,7 +109,7 @@ void shouldRunApp() { final String outputTopic = app.getTopics().getOutputTopic(); final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(outputTopic); - TestHelper.run(runner); + run(runner); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) @@ -135,7 +135,7 @@ void shouldUseMultipleLabeledInputTopics() { testClient.createTopic(inputTopic1); testClient.createTopic(inputTopic2); testClient.createTopic(outputTopic); - TestHelper.run(runner); + run(runner); testClient.send() .with(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class) .with(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class) @@ -161,7 +161,7 @@ void shouldThrowOnMissingInputTopic() { .stateListener(() -> this.stateListener) .uncaughtExceptionHandler(() -> this.uncaughtExceptionHandler) .build())) { - final Thread thread = TestHelper.run(runner); + final Thread thread = run(runner); final CapturingUncaughtExceptionHandler handler = (CapturingUncaughtExceptionHandler) thread.getUncaughtExceptionHandler(); awaitThreadIsDead(thread); @@ -184,7 +184,7 @@ void shouldCloseOnMapError() { final String outputTopic = app.getTopics().getOutputTopic(); final KafkaTestClient testClient = this.newTestClient(); testClient.createTopic(outputTopic); - final Thread thread = TestHelper.run(runner); + final Thread thread = run(runner); final CapturingUncaughtExceptionHandler handler = (CapturingUncaughtExceptionHandler) thread.getUncaughtExceptionHandler(); testClient.send() From 817f2d6b0a23d340aa3d78f0258f641e78ee19a1 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 20 Feb 2025 07:52:32 +0100 Subject: [PATCH 71/72] Update dependencies --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index e6d3c698..f3c1a8b7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,7 @@ org.gradle.parallel=true kafkaVersion=3.8.1 testContainersVersion=1.20.4 confluentVersion=7.8.0 -fluentKafkaVersion=3.0.1-SNAPSHOT +fluentKafkaVersion=3.1.0 junitVersion=5.11.4 mockitoVersion=5.15.2 assertJVersion=3.27.2 From 63ce55a851d200647bb92ba1250d198ad66a0df6 Mon Sep 17 00:00:00 2001 From: Philipp Schirmer Date: Thu, 20 Feb 2025 20:57:16 +0100 Subject: [PATCH 72/72] Add topics to TestTopology --- .../java/com/bakdata/kafka/KStreamXTest.java | 38 +++++++------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java index 0bd55297..8bdc17f2 100644 --- a/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java +++ b/streams-bootstrap-core/src/test/java/com/bakdata/kafka/KStreamXTest.java @@ -38,11 +38,8 @@ import java.time.Duration; import java.util.List; import java.util.Map; -import org.apache.kafka.common.serialization.Serde; import org.apache.kafka.common.serialization.Serdes; import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.StreamsConfig; -import org.apache.kafka.streams.TestOutputTopic; import org.apache.kafka.streams.errors.StreamsException; import org.apache.kafka.streams.kstream.Branched; import org.apache.kafka.streams.kstream.ForeachAction; @@ -238,17 +235,12 @@ public void buildTopology(final TopologyBuilder builder) { }; try (final TestTopology topology = app.startApp()) { topology.input().add("foo", "bar"); - final StreamsConfig streamsConfig = topology.getStreamsConfig(); - final Serde keySerde = (Serde) streamsConfig.defaultKeySerde(); - final Serde valueSerde = (Serde) streamsConfig.defaultValueSerde(); - final TestOutputTopic outputTopic = topology.getTestDriver() - .createOutputTopic("foo", keySerde.deserializer(), valueSerde.deserializer()); - this.softly.assertThat(outputTopic.readRecordsToList()) - .hasSize(1) - .anySatisfy(outputRecord -> { - this.softly.assertThat(outputRecord.getKey()).isEqualTo("foo"); - this.softly.assertThat(outputRecord.getValue()).isEqualTo("bar"); - }); + topology.getOutputTopics().add("foo"); + topology.streamOutput() + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); } } @@ -267,16 +259,14 @@ public void buildTopology(final TopologyBuilder builder) { .withKeySerde(Serdes.String()) .withValueSerde(Serdes.String()) .add("foo", "bar"); - final Serde keySerde = Serdes.String(); - final Serde valueSerde = Serdes.String(); - final TestOutputTopic outputTopic = topology.getTestDriver() - .createOutputTopic("foo", keySerde.deserializer(), valueSerde.deserializer()); - this.softly.assertThat(outputTopic.readRecordsToList()) - .hasSize(1) - .anySatisfy(outputRecord -> { - this.softly.assertThat(outputRecord.getKey()).isEqualTo("foo"); - this.softly.assertThat(outputRecord.getValue()).isEqualTo("bar"); - }); + topology.getOutputTopics().add("foo"); + topology.streamOutput() + .withKeySerde(Serdes.String()) + .withValueSerde(Serdes.String()) + .expectNextRecord() + .hasKey("foo") + .hasValue("bar") + .expectNoMoreRecord(); } }