forked from raystack/dagger
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: Add dynamic source and sink kafka properties (#31)
* add kafka security module * aDD SASL class callback config for producer and consumer * Add config map * remove build.gradle * Add dynamic props * Update regex * rename var * Remove redundant imports * Rename prefix * Remove unused import * Update test * Add implementation for sink dynamic props * Add null checking for the additional props * Added validations on source config * Add docs and refactor pattern to enum * chECKSTYLE * Add readme * Make the pattern more specific and embedded to the enum * Add more test * bump version * Add unit tests * Use expected annotation * Assert exception message. Add fail mechanism in case of not throwing any exception * Use rule for asserting exception * Add more test case * add more unit test * feat: Enable multiple underscore parsing * test: Add test on multiple underscore parsing
- Loading branch information
1 parent
6d225b4
commit d7b2709
Showing
12 changed files
with
434 additions
and
6 deletions.
There are no files selected for viewing
18 changes: 18 additions & 0 deletions
18
...re/src/main/java/com/gotocompany/dagger/core/enumeration/KafkaConnectorTypesMetadata.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package com.gotocompany.dagger.core.enumeration; | ||
|
|
||
| import java.util.regex.Pattern; | ||
|
|
||
| public enum KafkaConnectorTypesMetadata { | ||
| SOURCE("SOURCE_KAFKA_CONSUMER_CONFIG_+"), SINK("SINK_KAFKA_PRODUCER_CONFIG_+"); | ||
|
|
||
| KafkaConnectorTypesMetadata(String prefixPattern) { | ||
| this.prefixPattern = prefixPattern; | ||
| } | ||
|
|
||
| private final String prefixPattern; | ||
|
|
||
| public Pattern getConfigurationPattern() { | ||
| return Pattern.compile(String.format("^%s(.*)", prefixPattern), Pattern.CASE_INSENSITIVE); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
...dagger/core/source/config/adapter/DaggerKafkaConsumerAdditionalConfigurationsAdaptor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package com.gotocompany.dagger.core.source.config.adapter; | ||
|
|
||
| import com.google.gson.Gson; | ||
| import com.google.gson.TypeAdapter; | ||
| import com.google.gson.stream.JsonReader; | ||
| import com.google.gson.stream.JsonWriter; | ||
| import com.gotocompany.dagger.core.enumeration.KafkaConnectorTypesMetadata; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| public class DaggerKafkaConsumerAdditionalConfigurationsAdaptor extends TypeAdapter<Map<String, String>> { | ||
|
|
||
| @Override | ||
| public void write(JsonWriter jsonWriter, Map<String, String> stringStringMap) throws IOException { | ||
| Gson gson = new Gson(); | ||
| jsonWriter.jsonValue(gson.toJson(stringStringMap)); | ||
| } | ||
|
|
||
| @Override | ||
| public Map<String, String> read(JsonReader jsonReader) throws IOException { | ||
| Gson gson = new Gson(); | ||
| Map<String, String> map = gson.fromJson(jsonReader, Map.class); | ||
| List<String> invalidProps = map.keySet().stream() | ||
| .filter(key -> !KafkaConnectorTypesMetadata.SOURCE.getConfigurationPattern() | ||
| .matcher(key) | ||
| .matches()) | ||
| .collect(Collectors.toList()); | ||
| if (!invalidProps.isEmpty()) { | ||
| throw new IllegalArgumentException("Invalid additional kafka consumer configuration properties found: " + invalidProps); | ||
| } | ||
| return map.entrySet() | ||
| .stream() | ||
| .filter(entry -> entry.getValue() != null) | ||
| .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
28 changes: 28 additions & 0 deletions
28
dagger-core/src/main/java/com/gotocompany/dagger/core/utils/KafkaConfigUtil.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package com.gotocompany.dagger.core.utils; | ||
|
|
||
| import com.gotocompany.dagger.core.enumeration.KafkaConnectorTypesMetadata; | ||
|
|
||
| import java.util.Properties; | ||
| import java.util.Set; | ||
| import java.util.regex.Matcher; | ||
|
|
||
| public class KafkaConfigUtil { | ||
|
|
||
| public static Properties parseKafkaConfiguration(KafkaConnectorTypesMetadata kafkaConnectorTypesMetadata, Properties properties) { | ||
| Properties kafkaProperties = new Properties(); | ||
| Set<Object> configKeys = properties.keySet(); | ||
|
|
||
| for (Object key : configKeys) { | ||
| Matcher matcher = kafkaConnectorTypesMetadata.getConfigurationPattern() | ||
| .matcher(key.toString()); | ||
| if (matcher.find()) { | ||
| String kafkaConfigKey = matcher.group(1) | ||
| .toLowerCase() | ||
| .replaceAll("_+", "."); | ||
| kafkaProperties.setProperty(kafkaConfigKey, properties.get(key).toString()); | ||
| } | ||
| } | ||
| return kafkaProperties; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.