-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added a JSON payload formatter (#22)
* Initial working version of JsonPayloadFormatter and unit tests * make JsonPayloadFormatter Configurable * Add key/value schema visibility control * Use logging for formatter test output; configure with src/test/resources/simplelogger.properties * Fix schema visibility * Java import clean-up * Test lambda function dumps event as json * Add JsonPayloadFormatter description to README * Restored connect node * Use enum for visibility; updates from review comments * Use overloaded methods * Add support for batch in JsonPayloadFormatter * Clean up Invocation payloads section * Add integer, long, boolean key tests; remove some constants in tests as it actually made the code less understandable * Cleaned up example avro schema * Remove testing artifact names from example * v1.0.0
- Loading branch information
1 parent
0c9caca
commit 0ece5c1
Showing
15 changed files
with
1,526 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
129 changes: 129 additions & 0 deletions
129
src/main/java/com/nordstrom/kafka/connect/formatters/JsonPayloadFormatter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| package com.nordstrom.kafka.connect.formatters; | ||
|
|
||
| import com.fasterxml.jackson.core.JsonProcessingException; | ||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.ObjectWriter; | ||
| import org.apache.kafka.common.Configurable; | ||
| import org.apache.kafka.connect.data.Schema; | ||
| import org.apache.kafka.connect.json.JsonConverter; | ||
| import org.apache.kafka.connect.json.JsonConverterConfig; | ||
| import org.apache.kafka.connect.json.JsonDeserializer; | ||
| import org.apache.kafka.connect.sink.SinkRecord; | ||
|
|
||
| import java.util.Collection; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| import static java.util.Collections.emptyMap; | ||
|
|
||
| public class JsonPayloadFormatter implements PayloadFormatter, Configurable { | ||
| enum SchemaVisibility { | ||
| ALL, | ||
| MIN, | ||
| NONE | ||
| } | ||
|
|
||
| private final ObjectWriter recordWriter = new ObjectMapper().writerFor(Payload.class); | ||
| private final ObjectWriter recordsWriter = new ObjectMapper().writerFor(Payload[].class); | ||
| private final JsonConverter converter = new JsonConverter(); | ||
| private final JsonConverter converterSansSchema = new JsonConverter(); | ||
| private final JsonDeserializer deserializer = new JsonDeserializer(); | ||
| private SchemaVisibility keySchemaVisibility = SchemaVisibility.MIN; | ||
| private SchemaVisibility valueSchemaVisibility = SchemaVisibility.MIN; | ||
|
|
||
| public JsonPayloadFormatter() { | ||
| converter.configure(emptyMap(), false); | ||
|
|
||
| Map<String, String> configs = new HashMap<>(); | ||
| configs.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false"); | ||
| converterSansSchema.configure(configs, false); | ||
|
|
||
| deserializer.configure(emptyMap(), false); | ||
| } | ||
|
|
||
| @Override | ||
| public void configure(Map<String, ?> configs) { | ||
| keySchemaVisibility = configureSchemaVisibility(configs, "formatter.key.schema.visibility"); | ||
| valueSchemaVisibility = configureSchemaVisibility(configs, "formatter.value.schema.visibility"); | ||
| } | ||
|
|
||
| private SchemaVisibility configureSchemaVisibility(final Map<String, ?> configs, final String key) { | ||
| SchemaVisibility viz = SchemaVisibility.MIN; | ||
| final Object visibility = configs.get(key); | ||
| if (visibility != null) { | ||
| switch (visibility.toString()) { | ||
| case "all": | ||
| viz = SchemaVisibility.ALL; | ||
| break; | ||
| case "min": | ||
| viz = SchemaVisibility.MIN; | ||
| break; | ||
| case "none": | ||
| viz = SchemaVisibility.NONE; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| return viz; | ||
| } | ||
|
|
||
| public String format(final SinkRecord record) { | ||
| try { | ||
| return recordWriter.writeValueAsString(recordToPayload(record)); | ||
| } catch (JsonProcessingException e) { | ||
| throw new PayloadFormattingException(e); | ||
| } | ||
| } | ||
|
|
||
| public String format(final Collection<SinkRecord> records) { | ||
| final Payload[] payloads = records | ||
| .stream() | ||
| .map(this::recordToPayload) | ||
| .toArray(Payload[]::new); | ||
|
|
||
| try { | ||
| return recordsWriter.writeValueAsString(payloads); | ||
| } catch (final JsonProcessingException e) { | ||
| throw new PayloadFormattingException(e); | ||
| } | ||
| } | ||
|
|
||
| private Payload<Object, Object> recordToPayload(final SinkRecord record) { | ||
| Object deserializedKey; | ||
| Object deserializedValue; | ||
| if (record.keySchema() == null) { | ||
| deserializedKey = record.key(); | ||
| } else { | ||
| deserializedKey = deserialize(keySchemaVisibility, record.topic(), record.keySchema(), record.key()); | ||
| } | ||
| if (record.valueSchema() == null) { | ||
| deserializedValue = record.value(); | ||
| } else { | ||
| deserializedValue = deserialize(valueSchemaVisibility, record.topic(), record.valueSchema(), record.value()); | ||
| } | ||
|
|
||
| Payload<Object, Object> payload = new Payload<>(record); | ||
| payload.setKey(deserializedKey); | ||
| payload.setValue(deserializedValue); | ||
| if (keySchemaVisibility == SchemaVisibility.NONE) { | ||
| payload.setKeySchemaName(null); | ||
| payload.setKeySchemaVersion(null); | ||
| } | ||
| if (valueSchemaVisibility == SchemaVisibility.NONE) { | ||
| payload.setValueSchemaName(null); | ||
| payload.setValueSchemaVersion(null); | ||
| } | ||
|
|
||
| return payload; | ||
| } | ||
|
|
||
| private JsonNode deserialize(final SchemaVisibility schemaVisibility, final String topic, final Schema schema, final Object value) { | ||
| if (schemaVisibility == SchemaVisibility.ALL) { | ||
| return deserializer.deserialize(topic, converter.fromConnectData(topic, schema, value)); | ||
| } else { | ||
| return deserializer.deserialize(topic, converterSansSchema.fromConnectData(topic, schema, value)); | ||
| } | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.