From 3ad5e53d080eba43bdc6e0f58c2b331c8fe4c75e Mon Sep 17 00:00:00 2001 From: dardanxh Date: Fri, 18 Dec 2020 01:30:19 +0100 Subject: [PATCH] Implement JMS Source/Sink Plugin --- .gitignore | 36 ++ checkstyle.xml | 406 ++++++++++++++++++ docs/JMS-batchsink.md | 58 +++ docs/JMS-streamingsource.md | 102 +++++ examples/JMS-batchsink.json | 107 +++++ examples/JMS-streamingsource.json | 99 +++++ pom.xml | 230 ++++++++++ .../io/cdap/plugin/jms/common/JMSConfig.java | 172 ++++++++ .../cdap/plugin/jms/common/JMSConnection.java | 343 +++++++++++++++ .../plugin/jms/common/JMSDataStructures.java | 25 ++ .../plugin/jms/common/JMSMessageHeader.java | 136 ++++++ .../plugin/jms/common/JMSMessageParts.java | 33 ++ .../jms/common/JMSMessageProperties.java | 132 ++++++ .../plugin/jms/common/JMSMessageType.java | 28 ++ .../jms/common/SchemaValidationUtils.java | 328 ++++++++++++++ .../io/cdap/plugin/jms/sink/JMSBatchSink.java | 82 ++++ .../plugin/jms/sink/JMSBatchSinkConfig.java | 114 +++++ .../cdap/plugin/jms/sink/JMSOutputFormat.java | 71 +++ .../jms/sink/JMSOutputFormatProvider.java | 50 +++ .../cdap/plugin/jms/sink/JMSRecordWriter.java | 98 +++++ .../RecordToBytesMessageConverter.java | 75 ++++ .../RecordToMapMessageConverter.java | 75 ++++ .../converters/RecordToMessageConverter.java | 76 ++++ .../RecordToObjectMessageConverter.java | 55 +++ .../RecordToTextMessageConverter.java | 90 ++++ .../SinkMessageConverterFacade.java | 75 ++++ .../cdap/plugin/jms/source/JMSReceiver.java | 105 +++++ .../plugin/jms/source/JMSSourceUtils.java | 42 ++ .../plugin/jms/source/JMSStreamingSource.java | 76 ++++ .../jms/source/JMSStreamingSourceConfig.java | 225 ++++++++++ .../jms/source/ReferenceStreamingSource.java | 48 +++ .../BytesMessageToRecordConverter.java | 211 +++++++++ .../MapMessageToRecordConverter.java | 147 +++++++ .../converters/MessageToRecordConverter.java | 55 +++ .../ObjectMessageToRecordConverter.java | 62 +++ .../SourceMessageConverterFacade.java | 56 +++ .../TextMessageToRecordConverter.java | 56 +++ .../jms/common/SchemaValidationUtilsTest.java | 261 +++++++++++ .../RecordToBytesMessageConverterTest.java | 59 +++ .../RecordToMapMessageConverterTest.java | 54 +++ .../RecordToMessageConverterTest.java | 55 +++ .../RecordToObjectMessageConverterTest.java | 62 +++ .../RecordToTextMessageConverterTest.java | 71 +++ .../BytesMessageToRecordConverterTest.java | 97 +++++ .../MapMessageToRecordConverterTest.java | 95 ++++ .../MessageToRecordConverterTest.java | 93 ++++ .../ObjectMessageToRecordConverterTest.java | 56 +++ .../TextMessageToRecordConverterTest.java | 120 ++++++ .../source/utils/BytesMessageTestUtils.java | 58 +++ .../jms/source/utils/CommonTestUtils.java | 86 ++++ .../plugin/jms/source/utils/DummyObject.java | 63 +++ .../jms/source/utils/HeaderTestUtils.java | 71 +++ .../jms/source/utils/MapMessageTestUtils.java | 79 ++++ .../jms/source/utils/PropertiesTestUtils.java | 96 +++++ suppressions.xml | 35 ++ widgets/JMS-batchsink.json | 101 +++++ widgets/JMS-streamingsource.json | 207 +++++++++ 57 files changed, 5998 insertions(+) create mode 100644 .gitignore create mode 100644 checkstyle.xml create mode 100644 docs/JMS-batchsink.md create mode 100644 docs/JMS-streamingsource.md create mode 100644 examples/JMS-batchsink.json create mode 100644 examples/JMS-streamingsource.json create mode 100644 pom.xml create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSConfig.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSConnection.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSDataStructures.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSMessageHeader.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSMessageParts.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSMessageProperties.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/JMSMessageType.java create mode 100644 src/main/java/io/cdap/plugin/jms/common/SchemaValidationUtils.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/JMSBatchSink.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/JMSBatchSinkConfig.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormat.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormatProvider.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/JMSRecordWriter.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/sink/converters/SinkMessageConverterFacade.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/JMSReceiver.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/JMSSourceUtils.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/JMSStreamingSource.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/JMSStreamingSourceConfig.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/ReferenceStreamingSource.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverter.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/converters/SourceMessageConverterFacade.java create mode 100644 src/main/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverter.java create mode 100644 src/test/java/io/cdap/plugin/jms/common/SchemaValidationUtilsTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverterTest.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/utils/BytesMessageTestUtils.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/utils/CommonTestUtils.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/utils/DummyObject.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/utils/HeaderTestUtils.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/utils/MapMessageTestUtils.java create mode 100644 src/test/java/io/cdap/plugin/jms/source/utils/PropertiesTestUtils.java create mode 100644 suppressions.xml create mode 100644 widgets/JMS-batchsink.json create mode 100644 widgets/JMS-streamingsource.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b0df7b7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,36 @@ +*.class +.*.swp +.beamer +# Package Files # +*.jar +*.war +*.ear + +# Intellij Files & Dir # +*.iml +*.ipr +*.iws +atlassian-ide-plugin.xml +out/ +.DS_Store +./lib/ +.idea + +# Gradle Files & Dir # +build/ +.gradle/ +.stickyStorage +.build/ +target/ + +# Node log +npm-*.log +logs/ + +# Singlenode and test data files. +/templates/ +/data/ +/data-fabric-tests/data/ + +# generated by docs build +*.pyc diff --git a/checkstyle.xml b/checkstyle.xml new file mode 100644 index 0000000..2b6e777 --- /dev/null +++ b/checkstyle.xml @@ -0,0 +1,406 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/JMS-batchsink.md b/docs/JMS-batchsink.md new file mode 100644 index 0000000..74ccae6 --- /dev/null +++ b/docs/JMS-batchsink.md @@ -0,0 +1,58 @@ +# JMS batch sink + + +Description +----------- +Produces JMS messages of different types as Message, Text, Bytes and Map to a specified Queue or Topic. + +Use Case +-------- +Use this JMS Sink plugin when you want to produce messages to a JMS Queue/Topic. + + +Properties +---------- +**Connection Factory**: Name of the connection factory. If not specified, the value *ConnectionFactory* is considered by + default. + +**JMS Username**: Username to connect to JMS. This property is mandatory. + +**JMS Password**: Password to connect to JMS. This property is mandatory. + +**Provider URL**: Provider URL of the JMS Provider. This property is mandatory. For example for the *ActiveMQ* provider + you can set: `tcp://hostname:61616`. + +**Type**: Queue or Topic. Queue is considered by default. + +**Destination**: Queue/Topic name. + +**JNDI Context Factory**: Name of the context factory. This property is optional. For example for the *ActiveMQ* +provider you can set: `org.apache.activemq.jndi.ActiveMQInitialContextFactory`. + +**JNDI Username**: Username for JNDI. This property is optional. + +**JNDI Password**: Password for JNDI. This property is optional. + +**Message Type**: The type of messages you intend to produce. A JMS message could be of the following types: *Message*, + *Text*, *Bytes* and *Map*. By default, *Text* message type is considered. + +Example +------- +This example reads a record from a JSON file and produces a *MapMessage* with the file's content as a payload. +For every record of the JSON file it produces one *MapMessage* to the given topic. +An example of a JSON record is shown below. + +```json +{"name":"foo", "surname":"bar", "age":23} +``` +We could see the produced message in the running JMS implementation. + +| field name | value | +| ----------------- | ------------------------------------------ | +| messageId | ID:Producer-36705-1609122138230-1:1:1:1:1 | +| messageTimestamp | 1609122138554 | +| destination | topic://MyTopic | +| deliveryNode | 2 | +| expiration | 0 | +| priority | 4 | +| payload | {"name":"foo","surname":"bar","age":23} | diff --git a/docs/JMS-streamingsource.md b/docs/JMS-streamingsource.md new file mode 100644 index 0000000..38a4101 --- /dev/null +++ b/docs/JMS-streamingsource.md @@ -0,0 +1,102 @@ +# JMS Streaming Source + + +Description +----------- +Consumes JMS messages of different types as Message, Text, Bytes and Map from a specified Queue or Topic. + +Use Case +-------- +Use this JMS Source plugin when you want to consume messages from a JMS Queue/Topic and write them to a Table. + + +Properties +---------- +**Connection Factory**: Name of the connection factory. If not specified, the value *ConnectionFactory* is considered by + default. + +**JMS Username**: Username to connect to JMS. This property is mandatory. + +**JMS Password**: Password to connect to JMS. This property is mandatory. + +**Provider URL**: Provider URL of the JMS Provider. This property is mandatory. For example for the *ActiveMQ* provider + you can set: `tcp://hostname:61616`. + +**Type**: Queue or Topic. Queue is considered by default. + +**Destination**: Queue/Topic name. + +**JNDI Context Factory**: Name of the context factory. This property is optional. For example for the *ActiveMQ* +provider you can set: `org.apache.activemq.jndi.ActiveMQInitialContextFactory`. + +**JNDI Username**: Username for JNDI. This property is optional. + +**JNDI Password**: Password for JNDI. This property is optional. + +**Message Type**: The type of messages you intend to consume. A JMS message could be of the following types: *Message*, + *Text*, *Bytes* and *Map*. By default, *Text* message type is considered. The *payload* field of the output schema gets + switched to the appropriate data type upon the selection of a message type and *validate* button click. + + +Example +------- +This example reads JMS messages of *TextMessage* type from the *status* topic existing in provider *tcp://hostname +:616161* with JNDI context factory name *org.apache.activemq.jndi.ActiveMQInitialContextFactory*. An example of a +TextMessage object is shown below. Since we used ActiveMQ as a provider, the message is automatically considered as an +*ActiveMQTextMessage* (an implementation of JMS TextMessage interface). The object below shows an +*ActiveMQTextMessage* consumed: + +```text +ActiveMQTextMessage { + commandId=5, + responseRequired=true, + messageId=ID: Producer-50444-1608735228752-1: 1: 1: 1: 1, + originalDestination=null, + originalTransactionId=null, + producerId=ID: Producer-50444-1608735228752-1: 1: 1: 1, + destination=topic: topic://status, + transactionId=null, + expiration=0, + timestamp=1608735228894, + arrival=0, + brokerInTime=1608735228895, + brokerOutTime=1608735228896, + correlationId=null, + replyTo=null, + persistent=true, + type=null, + priority=4, + groupID=null, + groupSequence=0, + targetConsumerId=null, + compressed=false, + userID=null, + content=org.apache.activemq.util.ByteSequence@4b9e255, + marshalledProperties=null, + dataStructure=null, + redeliveryCounter=0, + size=0, + properties=null, + readOnlyProperties=true, + readOnlyBody=true, + droppable=false, + jmsXGroupFirstForConsumer=false, + text=DONE +} + +``` +Since the JMS Source plugin's implementation is purely based in JMS (ie. not coupled in any JMS implementation as for +example ActiveMQ), we consider only the header data supported by JMS. The consumed will output the below +record: + +| field name | value | +| ----------------- | ------------------------------------------ | +| messageId | ID:Producer-54511-1608749039578-1:1:1:1:1 | +| messageTimestamp | 1609122138554 | +| deliveryNode | 2 | +| payload | DONE | +| replyTo | topic://status | +| correlationId | null | +| expiration | 0 | +| type | null | +| redelivered | false | diff --git a/examples/JMS-batchsink.json b/examples/JMS-batchsink.json new file mode 100644 index 0000000..bbe2c24 --- /dev/null +++ b/examples/JMS-batchsink.json @@ -0,0 +1,107 @@ +{ + "artifact": { + "name": "cdap-data-pipeline", + "version": "6.3.0-SNAPSHOT", + "scope": "SYSTEM" + }, + "description": "Read records from a json file and generate a MapMessage for each one of them.", + "name": "JMS-batchsink", + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "File", + "to": "JMS" + } + ], + "comments": [], + "postActions": [], + "properties": {}, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "File", + "plugin": { + "name": "File", + "type": "batchsource", + "label": "File", + "artifact": { + "name": "core-plugins", + "version": "2.6.0-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "referenceName": "JMS", + "path": "${PATH}", + "format": "json", + "delimiter": ",", + "skipHeader": "true", + "filenameOnly": "false", + "recursive": "false", + "ignoreNonExistingFolders": "false", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"surname\",\"type\":\"string\"},{\"name\":\"age\",\"type\":\"int\"}]}" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"surname\",\"type\":\"string\"},{\"name\":\"age\",\"type\":\"int\"}]}" + } + ], + "id": "File" + }, + { + "name": "JMS", + "plugin": { + "name": "JMS", + "type": "batchsink", + "label": "JMS", + "artifact": { + "name": "jms-plugins", + "version": "1.0.0-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "referenceName": "JMS", + "type": "Topic", + "messageType": "Map", + "jmsUsername": "${JMS_USERNAME}", + "jmsPassword": "${JMS_PASSWORD}", + "providerUrl": "${PROVIDER_URL}", + "destination": "${DESTINATION}", + "jndiContextFactory": "${JNDI_CONTEXT_FACTORY}", + "jndiUsername": "${JNDI_USERNAME}", + "jndiPassword": "${JNDI_PASSWORD}" + } + }, + "outputSchema": [ + { + "name": "etlSchemaBody", + "schema": "" + } + ], + "inputSchema": [ + { + "name": "File", + "schema": "{\"type\":\"record\",\"name\":\"etlSchemaBody\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"surname\",\"type\":\"string\"},{\"name\":\"age\",\"type\":\"int\"}]}" + } + ], + "id": "JMS" + } + ], + "schedule": "0 * * * *", + "engine": "spark", + "numOfRecordsPreview": 100, + "description": "Data Pipeline Application", + "maxConcurrentRuns": 1 + } +} + diff --git a/examples/JMS-streamingsource.json b/examples/JMS-streamingsource.json new file mode 100644 index 0000000..05fde74 --- /dev/null +++ b/examples/JMS-streamingsource.json @@ -0,0 +1,99 @@ +{ + "artifact": { + "name": "cdap-data-streams", + "version": "6.3.0-SNAPSHOT", + "scope": "SYSTEM" + }, + "description": "Consume messages from a given topic and write them to a file.", + "name": "JMS-streamingsource", + "config": { + "resources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "driverResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "connections": [ + { + "from": "JMS", + "to": "File" + } + ], + "comments": [], + "postActions": [], + "properties": { + "system.spark.spark.streaming.backpressure.enabled": "true", + "system.spark.spark.executor.instances": "1" + }, + "processTimingEnabled": true, + "stageLoggingEnabled": false, + "stages": [ + { + "name": "JMS", + "plugin": { + "name": "JMS", + "type": "streamingsource", + "label": "JMS", + "artifact": { + "name": "jms-plugins", + "version": "1.0.0-SNAPSHOT", + "scope": "USER" + }, + "properties": { + "referenceName": "JMS", + "schema": "{\"type\":\"record\",\"name\":\"message\",\"fields\":[{\"name\":\"messageId\",\"type\":\"string\"},{\"name\":\"messageTimestamp\",\"type\":\"long\"},{\"name\":\"correlationId\",\"type\":\"string\"},{\"name\":\"replyTo\",\"type\":\"string\"},{\"name\":\"destination\",\"type\":\"string\"},{\"name\":\"deliveryNode\",\"type\":\"int\"},{\"name\":\"redelivered\",\"type\":\"boolean\"},{\"name\":\"type\",\"type\":\"string\"},{\"name\":\"expiration\",\"type\":\"long\"},{\"name\":\"priority\",\"type\":\"int\"},{\"name\":\"payload\",\"type\":\"string\"}]}", + "type": "Topic", + "messageType": "Text", + "jmsUsername": "${JMS_USERNAME}", + "jmsPassword": "${JMS_PASSWORD}", + "providerUrl": "${PROVIDER_URL}", + "destination": "${DESTINATION}", + "jndiContextFactory": "${JNDI_CONTEXT_FACTORY}" + } + }, + "outputSchema": "{\"type\":\"record\",\"name\":\"message\",\"fields\":[{\"name\":\"messageId\",\"type\":\"string\"},{\"name\":\"messageTimestamp\",\"type\":\"long\"},{\"name\":\"correlationId\",\"type\":\"string\"},{\"name\":\"replyTo\",\"type\":\"string\"},{\"name\":\"destination\",\"type\":\"string\"},{\"name\":\"deliveryNode\",\"type\":\"int\"},{\"name\":\"redelivered\",\"type\":\"boolean\"},{\"name\":\"type\",\"type\":\"string\"},{\"name\":\"expiration\",\"type\":\"long\"},{\"name\":\"priority\",\"type\":\"int\"},{\"name\":\"payload\",\"type\":\"string\"}]}", + "id": "JMS" + }, + { + "name": "File", + "plugin": { + "name": "File", + "type": "batchsink", + "label": "File", + "artifact": { + "name": "core-plugins", + "version": "2.5.0-SNAPSHOT", + "scope": "SYSTEM" + }, + "properties": { + "suffix": "yyyy-MM-dd-HH-mm", + "format": "delimited", + "referenceName": "file", + "path": "${PATH}", + "delimiter": ";", + "schema": "{\"name\":\"etlSchemaBody\",\"type\":\"record\",\"fields\":[{\"name\":\"body\",\"type\":\"string\"}]}" + } + }, + "outputSchema": "{\"name\":\"etlSchemaBody\",\"type\":\"record\",\"fields\":[{\"name\":\"body\",\"type\":\"string\"}]}", + "inputSchema": [ + { + "name": "JMS", + "schema": "{\"type\":\"record\",\"name\":\"message\",\"fields\":[{\"name\":\"messageId\",\"type\":\"string\"},{\"name\":\"messageTimestamp\",\"type\":\"long\"},{\"name\":\"correlationId\",\"type\":\"string\"},{\"name\":\"replyTo\",\"type\":\"string\"},{\"name\":\"destination\",\"type\":\"string\"},{\"name\":\"deliveryNode\",\"type\":\"int\"},{\"name\":\"redelivered\",\"type\":\"boolean\"},{\"name\":\"type\",\"type\":\"string\"},{\"name\":\"expiration\",\"type\":\"long\"},{\"name\":\"priority\",\"type\":\"int\"},{\"name\":\"payload\",\"type\":\"string\"}]}" + } + ], + "id": "File" + } + ], + "batchInterval": "10s", + "clientResources": { + "memoryMB": 2048, + "virtualCores": 1 + }, + "disableCheckpoints": true, + "stopGracefully": true, + "description": "Data Streams Application" + } +} + diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..6082a1e --- /dev/null +++ b/pom.xml @@ -0,0 +1,230 @@ + + + + + + 4.0.0 + JMS Plugins + io.cdap.plugin + jms-plugins + 1.0.0-SNAPSHOT + + + UTF-8 + widgets + docs + ${project.basedir} + 6.2.3 + 2.6.0-SNAPSHOT + 2.3.1 + 2.9.1 + 4.1.16.Final + 1.3.0 + 1.8.2 + 4.12 + 27.0.1-jre + 5.11.1 + 2.24.0 + + + + + org.apache.activemq + activemq-all + ${activemq.version} + + + io.cdap.plugin + hydrator-common + ${cdap.plugin.version} + + + com.google.guava + guava + ${guava.version} + + + io.cdap.cdap + cdap-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-common + ${cdap.version} + provided + + + io.cdap.cdap + cdap-api-common + ${cdap.version} + provided + + + io.cdap.cdap + cdap-etl-api-spark + ${cdap.version} + provided + + + io.cdap.cdap + cdap-etl-api + ${cdap.version} + provided + + + io.cdap.cdap + cdap-formats + ${cdap.version} + + + junit + junit + ${junit.version} + test + + + org.mockito + mockito-core + ${mockito.version} + test + + + io.cdap.cdap + hydrator-test + ${cdap.version} + test + + + org.apache.spark + spark-streaming_2.11 + ${spark.version} + provided + + + antlr + antlr + 2.7.7 + compile + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + 2.17 + + + validate + validate + + checkstyle.xml + suppressions.xml + UTF-8 + true + true + true + + + check + + + + + + com.puppycrawl.tools + checkstyle + 6.19 + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.7.0 + + 1.8 + 1.8 + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.14.1 + + -Xmx2048m -Djava.awt.headless=true -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC + -XX:OnOutOfMemoryError="kill -9 %p" -XX:+HeapDumpOnOutOfMemoryError + + false + + + + org.apache.felix + maven-bundle-plugin + 3.3.0 + + + <_exportcontents> + io.cdap.plugin.*; + org.apache.activemq.*; + org.apache.spark.streaming.*; + com.google.common.base.*; + + *;inline=false;scope=compile + true + lib + + + + + package + + bundle + + + + + + io.cdap + cdap-maven-plugin + 1.1.0 + + + system:cdap-data-pipeline[6.2.3, 7.0.0) + system:cdap-data-streams[6.2.3, 7.0.0) + + + + + create-artifact-config + prepare-package + + create-plugin-json + + + + + + + + diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSConfig.java b/src/main/java/io/cdap/plugin/jms/common/JMSConfig.java new file mode 100644 index 0000000..b9872a8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSConfig.java @@ -0,0 +1,172 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.common.ReferencePluginConfig; + +import javax.annotation.Nullable; + +/** + * Base config for JMS plugins. + */ +public class JMSConfig extends ReferencePluginConfig { + + public static final String NAME_CONNECTION_FACTORY = "connectionFactory"; + public static final String NAME_JMS_USERNAME = "jmsUsername"; + public static final String NAME_JMS_PASSWORD = "jmsPassword"; + public static final String NAME_PROVIDER_URL = "providerUrl"; + public static final String NAME_TYPE = "type"; + public static final String NAME_JNDI_CONTEXT_FACTORY = "jndiContextFactory"; + public static final String NAME_JNDI_USERNAME = "jndiUsername"; + public static final String NAME_JNDI_PASSWORD = "jndiPassword"; + + @Name(NAME_CONNECTION_FACTORY) + @Description("Name of the connection factory.") + @Nullable + @Macro + private String connectionFactory; // default: ConnectionFactory + + @Name(NAME_JMS_USERNAME) + @Description("Username to connect to JMS.") + @Macro + private String jmsUsername; + + @Name(NAME_JMS_PASSWORD) + @Description("Password to connect to JMS.") + @Macro + private String jmsPassword; + + @Name(NAME_PROVIDER_URL) + @Description("The URL of the JMS provider. For example, in case of an ActiveMQ Provider, the URL has the format " + + "tcp://hostname:61616.") + @Macro + private String providerUrl; + + @Name(NAME_TYPE) + @Description("Queue or Topic.") + @Nullable + @Macro + private String type; // default: queue + + @Name(NAME_JNDI_CONTEXT_FACTORY) + @Description("Name of the JNDI context factory. For example, in case of an ActiveMQ Provider, the JNDI Context " + + "Factory is: org.apache.activemq.jndi.ActiveMQInitialContextFactory.") + @Macro + private String jndiContextFactory; // default: org.apache.activemq.jndi.ActiveMQInitialContextFactory + + @Name(NAME_JNDI_USERNAME) + @Description("User name for the JNDI.") + @Nullable + @Macro + private String jndiUsername; + + @Name(NAME_JNDI_PASSWORD) + @Description("Password for the JNDI.") + @Nullable + @Macro + private String jndiPassword; + + public JMSConfig(String referenceName) { + super(referenceName); + this.connectionFactory = "ConnectionFactory"; + this.type = JMSDataStructures.QUEUE; + this.jndiContextFactory = "org.apache.activemq.jndi.ActiveMQInitialContextFactory"; + + } + + @VisibleForTesting + public JMSConfig(String referenceName, String connectionFactory, String jmsUsername, String jmsPassword, + String providerUrl, String type, String jndiContextFactory, String jndiUsername, + String jndiPassword) { + super(referenceName); + this.connectionFactory = connectionFactory; + this.jmsUsername = jmsUsername; + this.jmsPassword = jmsPassword; + this.providerUrl = providerUrl; + this.type = type; + this.jndiContextFactory = jndiContextFactory; + this.jndiUsername = jndiUsername; + this.jndiPassword = jndiPassword; + + } + + public String getConnectionFactory() { + return connectionFactory; + } + + public String getJmsUsername() { + return jmsUsername; + } + + public String getJmsPassword() { + return jmsPassword; + } + + public String getProviderUrl() { + return providerUrl; + } + + public String getType() { + return type; + } + + public String getJndiContextFactory() { + return jndiContextFactory; + } + + public String getJndiUsername() { + return jndiUsername; + } + + public String getJndiPassword() { + return jndiPassword; + } + + public void validateParams(FailureCollector failureCollector) { + + if (Strings.isNullOrEmpty(jmsUsername) && !containsMacro(NAME_JMS_USERNAME)) { + failureCollector + .addFailure("JMS username must be provided.", "Please provide your JMS username.") + .withConfigProperty(NAME_JMS_USERNAME); + } + + if (Strings.isNullOrEmpty(jmsPassword) && !containsMacro(NAME_JMS_PASSWORD)) { + failureCollector + .addFailure("JMS password must be provided.", "Please provide your JMS password.") + .withConfigProperty(NAME_JMS_PASSWORD); + } + + if (Strings.isNullOrEmpty(jndiContextFactory) && !containsMacro(NAME_JNDI_CONTEXT_FACTORY)) { + failureCollector + .addFailure("JNDI context factory must be provided.", "Please provide your JNDI" + + " context factory.") + .withConfigProperty(NAME_JNDI_CONTEXT_FACTORY); + } + + if (Strings.isNullOrEmpty(providerUrl) && !containsMacro(NAME_PROVIDER_URL)) { + failureCollector + .addFailure("Provider URL must be provided.", "Please provide your provider URL.") + .withConfigProperty(NAME_PROVIDER_URL); + } + } +} diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSConnection.java b/src/main/java/io/cdap/plugin/jms/common/JMSConnection.java new file mode 100644 index 0000000..c0b399b --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSConnection.java @@ -0,0 +1,343 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import com.google.common.base.Strings; +import io.cdap.plugin.jms.sink.JMSBatchSinkConfig; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Properties; +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.MessageConsumer; +import javax.jms.MessageListener; +import javax.jms.MessageProducer; +import javax.jms.Queue; +import javax.jms.Session; +import javax.jms.Topic; +import javax.naming.Context; +import javax.naming.InitialContext; +import javax.naming.NamingException; + +/** + * A facade class that encapsulates the necessary functionality to: get the initial context, resolve the connection + * factory, establish connection to JMS, create a session, resolve the destination by a queue or topic name, create + * producer, create consumer, set message listener to a consumer, and start connection. This class handles exceptions + * for all the functionalities provided. + */ +public class JMSConnection { + + private static final Logger LOG = LoggerFactory.getLogger(JMSConnection.class); + private final JMSConfig config; + + public JMSConnection(JMSConfig config) { + this.config = config; + } + + /** + * Gets the initial context by offering 'jndiContextFactory', 'providerUrl', 'topic'/'queue' name, `jndiUsername`, + * and `jndiPassword` config properties. + * + * @return the {@link InitialContext} from the given properties + */ + public Context getContext() { + Properties properties = new Properties(); + properties.put(Context.INITIAL_CONTEXT_FACTORY, config.getJndiContextFactory()); + properties.put(Context.PROVIDER_URL, config.getProviderUrl()); + + if (config instanceof JMSBatchSinkConfig) { + String destinationName = ((JMSBatchSinkConfig) config).getDestinationName(); + if (config.getType().equals(JMSDataStructures.TOPIC)) { + properties.put(String.format("topic.%s", destinationName), destinationName); + } else { + properties.put(String.format("queue.%s", destinationName), destinationName); + } + } else { + String sourceName = ((JMSStreamingSourceConfig) config).getSourceName(); + if (config.getType().equals(JMSDataStructures.TOPIC)) { + properties.put(String.format("topic.%s", sourceName), sourceName); + } else { + properties.put(String.format("queue.%s", sourceName), sourceName); + } + } + + if (!(Strings.isNullOrEmpty(config.getJndiUsername()) && Strings.isNullOrEmpty(config.getJndiPassword()))) { + properties.put(Context.SECURITY_PRINCIPAL, config.getJndiUsername()); + properties.put(Context.SECURITY_CREDENTIALS, config.getJndiPassword()); + } + + try { + return new InitialContext(properties); + } catch (NamingException e) { + throw new RuntimeException("Failed to create initial context for provider URL " + config.getProviderUrl() + + " with principal " + config.getJndiUsername(), e); + } + } + + /** + * Gets a {@link ConnectionFactory} by offering the `connectionFactory` config property. + * + * @param context an initial context + * @return a connection factory + */ + public ConnectionFactory getConnectionFactory(Context context) { + try { + return (ConnectionFactory) context.lookup(config.getConnectionFactory()); + } catch (NamingException e) { + throw new RuntimeException(String.format("Failed to resolve the connection factory for %s.", + config.getConnectionFactory()), e); + } + } + + /** + * Creates a {@link Connection} by offering `jmsUsername` and `jmsPassword`. If a source {@link Topic} is to be + * consumed set `clientId` which is needed by the JMS broker to identify the durable subscriber. + * + * @param connectionFactory a given connection factory + * @return a connection to the JMS broker + */ + public Connection createConnection(ConnectionFactory connectionFactory) { + Connection connection = null; + try { + connection = connectionFactory.createConnection(config.getJmsUsername(), config.getJmsPassword()); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + // If subscribing to a source topic, create a durable subscriber + if (config.getType().equals(JMSDataStructures.TOPIC)) { + try { + if (config instanceof JMSStreamingSourceConfig) { + String clientId = "client-id-" + ((JMSStreamingSourceConfig) config).getSourceName(); + connection.setClientID(clientId); + } + } catch (JMSException e) { + throw new RuntimeException("Cannot set Client Id", e); + } + } + return connection; + } + + /** + * Starts connection of this client to the JMS broker. + * + * @param connection a given connection + */ + public void startConnection(Connection connection) { + try { + connection.start(); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Stops connection of this client from the JMS broker. + * + * @param connection a given connection + */ + public void stopConnection(Connection connection) { + try { + connection.stop(); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Closes connection of this client from the JMS broker. + * + * @param connection a given connection + */ + public void closeConnection(Connection connection) { + try { + connection.close(); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Creates a {@link Session} between this client and the JMS broker. + * + * @param connection a given session + * @return a session to the JMS broker + */ + public Session createSession(Connection connection) { + try { + return connection.createSession(false, Session.AUTO_ACKNOWLEDGE); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Closes the {@link Session} between this client and the JMS broker. + * + * @param session a given session + */ + public void closeSession(Session session) { + try { + session.close(); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Gets the source {@link Topic}/{@link Queue} depending on the `type` config parameter. {@link Destination} is the + * parent class of {@link Topic} and {@link Queue}. + * + * @param context a given context + * @return a source topic/queue that this client is about to consume messages from + */ + public Destination getSource(Context context) { + String sourceName = ((JMSStreamingSourceConfig) config).getSourceName(); + if (config.getType().equals(JMSDataStructures.TOPIC)) { + try { + return (Topic) context.lookup(sourceName); + } catch (NamingException e) { + throw new RuntimeException("Failed to resolve the topic " + sourceName, e); + } + } else { + try { + return (Queue) context.lookup(sourceName); + } catch (NamingException e) { + throw new RuntimeException("Failed to resolve the queue " + sourceName, e); + } + } + } + + /** + * Gets a sink {@link Topic}/{@link Queue} depending on the `type` config parameter. {@link Destination} is the + * parent class of {@link Topic} and {@link Queue}. If no sink topic/queue name is provided, a sink topic/queue is + * automatically created. + * + * @param context a given context + * @param session a given session needed to create the topic/queue in case it does not exist + * @return a sink topic/queue this client is about to produce messages to + */ + public Destination getSink(Context context, Session session) { + String destinationName = ((JMSBatchSinkConfig) config).getDestinationName(); + + if (config.getType().equals(JMSDataStructures.TOPIC)) { + try { + return (Topic) context.lookup(destinationName); + } catch (NamingException e) { + LOG.warn("Failed to resolve queue " + destinationName, e); + return createSinkTopic(session); + } + } else { + try { + return (Queue) context.lookup(destinationName); + } catch (NamingException e) { + LOG.warn("Failed to resolve queue " + destinationName, e); + return createSinkQueue(session); + } + } + } + + /** + * Creates a sink {@link Topic} + * + * @param session a given session + * @return a created topic + */ + private Destination createSinkTopic(Session session) { + String destinationName = ((JMSBatchSinkConfig) config).getDestinationName(); + LOG.info("Creating topic " + destinationName); + try { + return session.createTopic(destinationName); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Creates a sink {@link Queue} + * + * @param session a given session + * @return a created queue + */ + private Destination createSinkQueue(Session session) { + String destinationName = ((JMSBatchSinkConfig) config).getDestinationName(); + LOG.info("Creating queue " + destinationName); + try { + return session.createQueue(destinationName); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Creates a {@link MessageConsumer} that consumes messages from a defined source {@link Topic}/{@link Queue}. In case + * of a topic-consumer, a durable subscriber is created. A durable subscriber makes the JMS broker keep the state of + * the offset consumed. Hence this client can restart consuming messages from the last offset not read. + * + * @param session a given session + * @param destination a source topic/queue this client is about to consume messages from + * @return a created message consumer + */ + public MessageConsumer createConsumer(Session session, Destination destination) { + MessageConsumer messageConsumer = null; + try { + if (destination instanceof Topic) { + String subscriberId = "subscriber-id-" + ((JMSStreamingSourceConfig) config).getSourceName(); + messageConsumer = session.createDurableSubscriber((Topic) destination, subscriberId); + } else { + messageConsumer = session.createConsumer(destination); + } + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return messageConsumer; + } + + /** + * Creates a {@link MessageProducer} that produces messages to a defined sink {@link Topic}/{@link Queue}. + * + * @param session a given session + * @param destination a sink topic/queue this client is about to produce messages to + * @return a created message producer + */ + public MessageProducer createProducer(Session session, Destination destination) { + try { + return session.createProducer(destination); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + /** + * Sets a {@link MessageListener} to the {@link MessageConsumer}. This message listener has a `onMessage()` method + * that gets automatically triggered in case a new message is produced to the queue/topic while the pipeline is in + * the RUNNING state. + * + * @param messageListener a given message listener + * @param messageConsumer a given message consumer + */ + public void setMessageListener(MessageListener messageListener, MessageConsumer messageConsumer) { + try { + messageConsumer.setMessageListener(messageListener); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } +} diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSDataStructures.java b/src/main/java/io/cdap/plugin/jms/common/JMSDataStructures.java new file mode 100644 index 0000000..4990af2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSDataStructures.java @@ -0,0 +1,25 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +/** + * A class that specifies the JMS data structures types. + */ +public class JMSDataStructures { + public static final String QUEUE = "Queue"; + public static final String TOPIC = "Topic"; +} diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSMessageHeader.java b/src/main/java/io/cdap/plugin/jms/common/JMSMessageHeader.java new file mode 100644 index 0000000..8146cb7 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSMessageHeader.java @@ -0,0 +1,136 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import javax.jms.JMSException; +import javax.jms.Message; + +/** + * A class that specifies JMS message header fields. + */ +public class JMSMessageHeader { + public static final String MESSAGE_ID = "messageId"; + public static final String MESSAGE_TIMESTAMP = "messageTimestamp"; + public static final String CORRELATION_ID = "correlationId"; + public static final String REPLY_TO = "replyTo"; + public static final String DESTINATION = "destination"; + public static final String DELIVERY_MODE = "deliveryNode"; + public static final String REDELIVERED = "redelivered"; + public static final String TYPE = "type"; + public static final String EXPIRATION = "expiration"; + public static final String PRIORITY = "priority"; + + public static Schema.Field getMessageHeaderField() { + return Schema.Field.of(JMSMessageParts.HEADER, Schema.recordOf( + JMSMessageParts.HEADER, + Schema.Field.of(MESSAGE_ID, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(MESSAGE_TIMESTAMP, Schema.nullableOf(Schema.of(Schema.Type.LONG))), + Schema.Field.of(CORRELATION_ID, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(REPLY_TO, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(DESTINATION, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(DELIVERY_MODE, Schema.nullableOf(Schema.of(Schema.Type.INT))), + Schema.Field.of(REDELIVERED, Schema.nullableOf(Schema.of(Schema.Type.BOOLEAN))), + Schema.Field.of(TYPE, Schema.nullableOf(Schema.of(Schema.Type.STRING))), + Schema.Field.of(EXPIRATION, Schema.nullableOf(Schema.of(Schema.Type.LONG))), + Schema.Field.of(PRIORITY, Schema.nullableOf(Schema.of(Schema.Type.INT))))); + } + + public static List getJMSMessageHeaderNames() { + return Arrays.asList(MESSAGE_ID, MESSAGE_TIMESTAMP, CORRELATION_ID, REPLY_TO, DESTINATION, DELIVERY_MODE, + REDELIVERED, TYPE, EXPIRATION, PRIORITY); + } + + public static String describe() { + return getJMSMessageHeaderNames().stream().collect(Collectors.joining(", ")); + } + + + /** + * Gets header data fields from the JMS message and adds them to the passed record builder. + * + * @param schema the entire schema of the record + * @param builder the record builder that we set the header record into + * @param message the incoming JMS message + * @throws JMSException in case the method fails to read fields from the JMS message + */ + public static void populateHeader(Schema schema, StructuredRecord.Builder builder, Message message) + throws JMSException { + Schema headerSchema = schema.getField(JMSMessageParts.HEADER).getSchema(); + StructuredRecord.Builder headerRecordBuilder = StructuredRecord.builder(headerSchema); + + for (Schema.Field field : headerSchema.getFields()) { + + switch (field.getName()) { + case JMSMessageHeader.MESSAGE_ID: + headerRecordBuilder.set(JMSMessageHeader.MESSAGE_ID, message.getJMSMessageID()); + break; + + case JMSMessageHeader.CORRELATION_ID: + headerRecordBuilder.set(JMSMessageHeader.CORRELATION_ID, message.getJMSCorrelationID()); + break; + + case JMSMessageHeader.REPLY_TO: + if (message.getJMSReplyTo() != null) { + headerRecordBuilder.set(JMSMessageHeader.REPLY_TO, message.getJMSReplyTo().toString()); + } else { + headerRecordBuilder.set(JMSMessageHeader.REPLY_TO, null); + } + break; + + case JMSMessageHeader.DESTINATION: + if (message.getJMSDestination() != null) { + headerRecordBuilder.set(JMSMessageHeader.DESTINATION, message.getJMSDestination().toString()); + } else { + headerRecordBuilder.set(JMSMessageHeader.DESTINATION, null); + } + break; + + case JMSMessageHeader.TYPE: + headerRecordBuilder.set(JMSMessageHeader.TYPE, message.getJMSType()); + break; + + case JMSMessageHeader.MESSAGE_TIMESTAMP: + headerRecordBuilder.set(JMSMessageHeader.MESSAGE_TIMESTAMP, message.getJMSTimestamp()); + break; + + case JMSMessageHeader.DELIVERY_MODE: + headerRecordBuilder.set(JMSMessageHeader.DELIVERY_MODE, message.getJMSDeliveryMode()); + break; + + case JMSMessageHeader.REDELIVERED: + headerRecordBuilder.set(JMSMessageHeader.REDELIVERED, message.getJMSRedelivered()); + break; + + case JMSMessageHeader.EXPIRATION: + headerRecordBuilder.set(JMSMessageHeader.EXPIRATION, message.getJMSExpiration()); + break; + + case JMSMessageHeader.PRIORITY: + headerRecordBuilder.set(JMSMessageHeader.PRIORITY, message.getJMSPriority()); + break; + } + } + builder.set(JMSMessageParts.HEADER, headerRecordBuilder.build()); + } +} + diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSMessageParts.java b/src/main/java/io/cdap/plugin/jms/common/JMSMessageParts.java new file mode 100644 index 0000000..26313ea --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSMessageParts.java @@ -0,0 +1,33 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import java.util.Arrays; +import java.util.List; + +/** + * A class that specifies JMS message parts. + */ +public class JMSMessageParts { + public static final String HEADER = "header"; + public static final String BODY = "body"; + public static final String PROPERTIES = "properties"; + + public static List getJMSMessageParts() { + return Arrays.asList(HEADER, BODY, PROPERTIES); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSMessageProperties.java b/src/main/java/io/cdap/plugin/jms/common/JMSMessageProperties.java new file mode 100644 index 0000000..8c80b32 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSMessageProperties.java @@ -0,0 +1,132 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import com.google.gson.Gson; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import javax.jms.JMSException; +import javax.jms.Message; + +/** + * A class that specifies JMS message properties fields. + */ +public class JMSMessageProperties { + + /** + * Adds properties values in the structured record. + * + * @param schema the structured record schema + * @param builder the structured record builder + * @param message the JMS message + * @param messageType the JMS message type + * @throws JMSException + */ + public static void populateProperties(Schema schema, StructuredRecord.Builder builder, Message message, + String messageType) throws JMSException { + Schema.Type propertiesFieldType = schema.getField(JMSMessageParts.PROPERTIES).getSchema().getType(); + + if (propertiesFieldType.equals(Schema.Type.STRING)) { + populatePropertiesOnStringSchema(builder, message); + } else if (propertiesFieldType.equals(Schema.Type.RECORD)) { + populatePropertiesOnRecordSchema(schema, builder, message, messageType); + } else { + throw new RuntimeException( + String.format("Failed to populate properties! Field %s can only support String or Record data types!", + JMSMessageParts.PROPERTIES) + ); + } + } + + + /** + * @param builder + * @param message + * @throws JMSException + */ + public static void populatePropertiesOnStringSchema(StructuredRecord.Builder builder, Message message) + throws JMSException { + HashMap properties = new HashMap<>(); + List listOfPropertyNames = Collections.list(message.getPropertyNames()); + + for (String propertyName : listOfPropertyNames) { + properties.put(propertyName, message.getObjectProperty(propertyName)); + } + + builder.set(JMSMessageParts.PROPERTIES, new Gson().toJson(properties)); + } + + + /** + * @param schema the entire schema of the record + * @param recordBuilder the record builder that we set the properties record into + * @param message the incoming JMS message + * @param messageType the incoming JMS message type + * @throws JMSException + */ + public static void populatePropertiesOnRecordSchema(Schema schema, StructuredRecord.Builder recordBuilder, + Message message, String messageType) throws JMSException { + Schema propertiesSchema = schema.getField(JMSMessageParts.PROPERTIES).getSchema(); + + StructuredRecord.Builder propertiesRecordBuilder = StructuredRecord.builder(propertiesSchema); + + for (Schema.Field field : propertiesSchema.getFields()) { + String name = field.getName(); + Schema.Type type = field.getSchema().getType(); + + if (!message.propertyExists(field.getName())) { + throw new RuntimeException( + String.format("Property \"%1$s\" does not exist in the incoming \"%2$s\" message! " + + "Make sure that you have specified a correct field name in the output schema that " + + "matches a property name in the incoming \"%2$s\" message.", field.getName(), messageType)); + } + + switch (type) { + case BOOLEAN: + propertiesRecordBuilder.set(name, message.getBooleanProperty(name)); + continue; + case BYTES: + propertiesRecordBuilder.set(name, message.getByteProperty(name)); + continue; + case INT: + propertiesRecordBuilder.set(name, message.getIntProperty(name)); +// short getShortProperty(String var1) throws JMSException; + continue; + case LONG: + propertiesRecordBuilder.set(name, message.getLongProperty(name)); + continue; + case FLOAT: + propertiesRecordBuilder.set(name, message.getFloatProperty(name)); + continue; + case DOUBLE: + propertiesRecordBuilder.set(name, message.getDoubleProperty(name)); + continue; + case STRING: + propertiesRecordBuilder.set(name, message.getStringProperty(name)); + continue; + default: + propertiesRecordBuilder.set(name, message.getObjectProperty(name)); + continue; + } + } + recordBuilder.set(JMSMessageParts.PROPERTIES, propertiesRecordBuilder.build()); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/common/JMSMessageType.java b/src/main/java/io/cdap/plugin/jms/common/JMSMessageType.java new file mode 100644 index 0000000..e1d2524 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/JMSMessageType.java @@ -0,0 +1,28 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +/** + * A class that specifies JMS message types. + */ +public class JMSMessageType { + public static final String MESSAGE = "Message"; + public static final String TEXT = "Text"; + public static final String BYTES = "Bytes"; + public static final String MAP = "Map"; + public static final String OBJECT = "Object"; +} diff --git a/src/main/java/io/cdap/plugin/jms/common/SchemaValidationUtils.java b/src/main/java/io/cdap/plugin/jms/common/SchemaValidationUtils.java new file mode 100644 index 0000000..fc644f4 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/common/SchemaValidationUtils.java @@ -0,0 +1,328 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import com.google.common.annotations.VisibleForTesting; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * A class that validates the schema. + */ +public class SchemaValidationUtils { + + @VisibleForTesting + public static final String + SCHEMA_IS_NULL_ERROR = "Schema is null!", + SCHEMA_IS_NULL_ACTION = "Please provide the schema.", + + NOT_SUPPORTED_ROOT_FIELDS_ERROR = "Not supported root fields in the schema!", + NOT_SUPPORTED_ROOT_FIELDS_ACTION = "Only \"header\", \"properties\" and \"body\" are supported as root fields.", + + BODY_NOT_IN_SCHEMA_ERROR = "The mandatory field \"body\" is missing in the schema!", + BODY_NOT_IN_SCHEMA_ACTION = "Please provide \"body\" field", + + WRONG_BODY_DATA_TYPE_IN_TEXT_MESSAGE_ERROR = "The field \"body\" has a not supported data type set!", + WRONG_BODY_DATA_TYPE_IN_TEXT_MESSAGE_ACTION = "When JMS \"Text\" message type is selected, the field \"body\" is" + + " mandatory to be of String datatype", + + WRONG_BODY_DATA_TYPE_IN_MAP_MESSAGE_ERROR = "The field \"body\" has a not supported data type set!", + WRONG_BODY_DATA_TYPE_IN_MAP_MESSAGE_ACTION = "When JMS \"Map\" message type is selected, the field \"body\" is" + + " mandatory to be of String or Record data type.", + + WRONG_BODY_DATA_TYPE_IN_BYTES_MESSAGE_ERROR = "The field \"body\" has a not supported data type set!", + WRONG_BODY_DATA_TYPE_IN_BYTES_MESSAGE_ACTION = "When JMS \"Bytes\" message type is selected, the field \"body\"" + + " is mandatory to be of String or Record data type.", + + WRONG_BODY_DATA_TYPE_IN_OBJECT_MESSAGE_ERROR = "The field \"body\" has a not supported data type set!", + WRONG_BODY_DATA_TYPE_IN_OBJECT_MESSAGE_ACTION = "When JMS \"Object\" message type is selected, the field " + + "\"body\" is mandatory to be of Array of Bytes data type.", + + WRONG_BODY_DATA_TYPE_FOR_HEADER_ERROR = "The field \"header\" has a not supported data type set!", + WRONG_BODY_DATA_TYPE_FOR_HEADER_ACTION = "The \"header\" field must be of Record data type.", + + NOT_SUPPORTED_FIELDS_IN_HEADER_RECORD_ERROR = "Not supported fields set in the header record!", + NOT_SUPPORTED_FIELDS_IN_HEADER_RECORD_ACTION = "The field \"header\" support only fields: " + + JMSMessageHeader.describe(), + + WRONG_PROPERTIES_DATA_TYPE_ERROR = "The field \"properties\" has a not supported data type set!", + WRONG_PROPERTIES_DATA_TYPE_ACTION = "The field \"properties\" is mandatory to be of String or Record data type.", + + NOT_SUPPORTED_ROOT_FIELDS_IN_MESSAGE_ERROR = "Not supported root fields in the schema!", + NOT_SUPPORTED_ROOT_FIELDS_IN_MESSAGE_ACTION = "JMS \"Message\" message type supports only \"header\" and " + + "\"properties\" as root fields.", + + HEADER_AND_PROPERTIES_MISSING_IN_MESSAGE_ERROR = "Fields \"Header\" and \"Properties\" are missing!", + HEADER_AND_PROPERTIES_MISSING_IN_MESSAGE_ACTION = "When JMS \"Message\" message type is selected, it is " + + "mandatory that either \"Header\" or \"Properties\" root fields to be present in schema. " + + "Set at least one of \"Keep Message Header\" or \"Keep Message Properties\" to true."; + + /** + * Throws an error if schema is null. + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateIfSchemaIsNull(Schema schema, FailureCollector collector) { + if (schema == null) { + tell(collector, SCHEMA_IS_NULL_ERROR, SCHEMA_IS_NULL_ACTION); + } + } + + /** + * Throws an error if the input schema contains any other root fields except of "header", "properties", and "body". + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateIfAnyNotSupportedRootFieldExists(Schema schema, FailureCollector collector) { + boolean areNonSupportedFieldsPresent = schema + .getFields() + .stream() + .map(field -> field.getName()) + .anyMatch(f -> !JMSMessageParts.getJMSMessageParts().contains(f)); + + if (areNonSupportedFieldsPresent) { + tell(collector, NOT_SUPPORTED_ROOT_FIELDS_ERROR, NOT_SUPPORTED_ROOT_FIELDS_ACTION); + } + } + + /** + * Throws an error if the input schema does not contain the root field "body". JMS "Message" type is the only message + * type allowed to have the schema without the root field "body". + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateIfBodyNotInSchema(Schema schema, FailureCollector collector) { + boolean noBodyInSchema = !schema + .getFields() + .stream() + .map(field -> field.getName()) + .collect(Collectors.toList()) + .contains(JMSMessageParts.BODY); + + if (noBodyInSchema) { + tell(collector, BODY_NOT_IN_SCHEMA_ERROR, BODY_NOT_IN_SCHEMA_ACTION); + } + } + + /** + * Throws an error if the root field "body" is not of type "string" when JMS "TextMessage" is selected. + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateTextMessageSchema(Schema schema, FailureCollector collector) { + Schema.Type type = schema.getField(JMSMessageParts.BODY).getSchema().getType(); + boolean isTypeString = type.equals(Schema.Type.STRING); + + if (!isTypeString) { + tell(collector, WRONG_BODY_DATA_TYPE_IN_TEXT_MESSAGE_ERROR, WRONG_BODY_DATA_TYPE_IN_TEXT_MESSAGE_ACTION); + } + } + + /** + * Throws an error if the root field "body" is not of type "string" or "record" when JMS "MapMessage" is selected. + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateMapMessageSchema(Schema schema, FailureCollector collector) { + Schema.Type type = schema.getField(JMSMessageParts.BODY).getSchema().getType(); + boolean isTypeString = type.equals(Schema.Type.STRING); + boolean isTypeRecord = type.equals(Schema.Type.RECORD); + + if (!isTypeString && !isTypeRecord) { + tell(collector, WRONG_BODY_DATA_TYPE_IN_MAP_MESSAGE_ERROR, WRONG_BODY_DATA_TYPE_IN_MAP_MESSAGE_ACTION); + } + } + + /** + * Throws an error if the input schema contains any other root fields except of "header", and "properties" when JMS + * "Message" type is selected. + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateMessageSchema(Schema schema, FailureCollector collector) { + List fieldNames = schema.getFields().stream().map(field -> field.getName()).collect(Collectors.toList()); + + boolean areNonSupportedRootFieldsPresent = false; + for (String fieldName: fieldNames) { + if (Arrays.asList(JMSMessageParts.PROPERTIES, JMSMessageParts.HEADER).contains(fieldName)) { + areNonSupportedRootFieldsPresent = true; + break; + } + } + + if (areNonSupportedRootFieldsPresent) { + tell(collector, NOT_SUPPORTED_ROOT_FIELDS_IN_MESSAGE_ERROR, NOT_SUPPORTED_ROOT_FIELDS_IN_MESSAGE_ACTION); + } + + boolean areHeaderAndPropertiesMissing = !fieldNames.contains(JMSMessageParts.HEADER) && + !fieldNames.contains(JMSMessageParts.PROPERTIES); + if (areHeaderAndPropertiesMissing) { + tell(collector, HEADER_AND_PROPERTIES_MISSING_IN_MESSAGE_ERROR, HEADER_AND_PROPERTIES_MISSING_IN_MESSAGE_ACTION); + } + } + + /** + * Throws an error if the root field "body" is not of type "array of bytes" when JMS "ObjectMessage" is selected. + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateObjectMessageSchema(Schema schema, FailureCollector collector) { + boolean shouldThrowError = true; + boolean isTypeArray = schema + .getField(JMSMessageParts.BODY) + .getSchema() + .getType() + .equals(Schema.Type.ARRAY); + + if (isTypeArray) { + boolean isSubTypeByte = schema + .getField(JMSMessageParts.BODY) + .getSchema() + .getComponentSchema() + .getType() + .equals(Schema.Type.BYTES); + + if (isSubTypeByte) { + shouldThrowError = false; + } + } + + if (shouldThrowError) { + tell(collector, WRONG_BODY_DATA_TYPE_IN_OBJECT_MESSAGE_ERROR, WRONG_BODY_DATA_TYPE_IN_OBJECT_MESSAGE_ACTION); + } + } + + /** + * Throws an error if the root field "body" is not of type "string" or "record" when JMS "BytesMessage" is selected. + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateBytesMessageSchema(Schema schema, FailureCollector collector) { + Schema.Type type = schema.getField(JMSMessageParts.BODY).getSchema().getType(); + boolean isTypeString = type.equals(Schema.Type.STRING); + boolean isTypeRecord = type.equals(Schema.Type.RECORD); + + if (!isTypeString && !isTypeRecord) { + tell(collector, WRONG_BODY_DATA_TYPE_IN_BYTES_MESSAGE_ERROR, WRONG_BODY_DATA_TYPE_IN_BYTES_MESSAGE_ACTION); + } + } + + /** + * Throws an error if the root field "header" is not of type "record". Throws an error also if the header record + * contains other fields except of "messageId", "messageTimestamp", "correlationId", "replyTo", "destination", + * "deliveryNode", "redelivered", "type", "expiration", and "priority". + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validateHeaderSchema(Schema schema, FailureCollector collector) { + + if (!isFieldPresent(schema, JMSMessageParts.HEADER)) { + return; + } + + boolean isTypeRecord = schema + .getField(JMSMessageParts.HEADER) + .getSchema() + .getType() + .equals(Schema.Type.RECORD); + + if (!isTypeRecord) { + tell(collector, WRONG_BODY_DATA_TYPE_FOR_HEADER_ERROR, WRONG_BODY_DATA_TYPE_FOR_HEADER_ACTION); + } + + boolean areNonSupportedHeaderFieldsPresent = schema + .getField(JMSMessageParts.HEADER) + .getSchema() + .getFields() + .stream() + .map(field -> field.getName()) + .anyMatch(f -> !JMSMessageHeader.getJMSMessageHeaderNames().contains(f)); + + if (areNonSupportedHeaderFieldsPresent) { + tell(collector, NOT_SUPPORTED_FIELDS_IN_HEADER_RECORD_ERROR, NOT_SUPPORTED_FIELDS_IN_HEADER_RECORD_ACTION); + } + } + + /** + * Throws an error if the root field "properties" is not of type "string" or "record". + * + * @param schema the user defined schema + * @param collector the failure collector + */ + public static void validatePropertiesSchema(Schema schema, FailureCollector collector) { + + if (!isFieldPresent(schema, JMSMessageParts.PROPERTIES)) { + return; + } + + Schema.Type type = schema.getField(JMSMessageParts.PROPERTIES).getSchema().getType(); + boolean isTypeString = type.equals(Schema.Type.STRING); + boolean isTypeRecord = type.equals(Schema.Type.RECORD); + + if (!isTypeString && !isTypeRecord) { + tell(collector, WRONG_PROPERTIES_DATA_TYPE_ERROR, WRONG_PROPERTIES_DATA_TYPE_ACTION); + } + } + + /** + * Throws an error and also add the error in the failure collector if one is provided. + * + * @param collector the failure collector + * @param errorMessage the error message + * @param correctiveAction the action that the user should perform to resolve the error + */ + public static void tell(FailureCollector collector, String errorMessage, String correctiveAction) { + String errorNature = "Error during schema validation"; + + if (collector != null) { + collector.addFailure(errorNature + ": " + errorMessage, correctiveAction) + .withConfigProperty(JMSStreamingSourceConfig.NAME_SCHEMA); + } else { + throw new RuntimeException(concatenate(errorNature + ": " + errorMessage, correctiveAction)); + } + } + + private static boolean isFieldPresent(Schema schema, String fieldName) { + return schema.getField(fieldName) != null; + } + + /** + * Concatenates two strings with a space in between + * + * @param left the left string + * @param right the right string + * @return the concatenated string + */ + @VisibleForTesting + public static String concatenate(String left, String right) { + return String.format("%s %s", left, right); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/JMSBatchSink.java b/src/main/java/io/cdap/plugin/jms/sink/JMSBatchSink.java new file mode 100644 index 0000000..baa2e21 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/JMSBatchSink.java @@ -0,0 +1,82 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.batch.Output; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.api.dataset.lib.KeyValue; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.cdap.etl.api.batch.BatchSinkContext; +import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.ReferenceBatchSink; +import org.apache.hadoop.io.NullWritable; + +import java.io.IOException; +import java.util.stream.Collectors; + +/** + * A class that produces {@link StructuredRecord} to a JMS Queue or Topic. + */ +@Plugin(type = BatchSink.PLUGIN_TYPE) +@Name("JMS") +@Description("JMSSink") +public class JMSBatchSink extends ReferenceBatchSink { + + private final JMSBatchSinkConfig config; + + public JMSBatchSink(JMSBatchSinkConfig config) { + super(config); + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + config.validateParams(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().getFailureCollector().getOrThrowException(); + } + + @Override + public void prepareRun(BatchSinkContext context) throws Exception { + LineageRecorder lineageRecorder = new LineageRecorder(context, config.referenceName); + Schema schema = context.getInputSchema(); + + if (schema != null) { + lineageRecorder.createExternalDataset(schema); + if (schema.getFields() != null && !schema.getFields().isEmpty()) { + lineageRecorder.recordWrite("Write", "Wrote to JMS topic.", + schema.getFields().stream() + .map(Schema.Field::getName) + .collect(Collectors.toList())); + } + } + + context.addOutput(Output.of(config.referenceName, new JMSOutputFormatProvider(config))); + } + + @Override + public void transform(StructuredRecord input, Emitter> emitter) + throws IOException { + emitter.emit(new KeyValue<>(null, input)); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/JMSBatchSinkConfig.java b/src/main/java/io/cdap/plugin/jms/sink/JMSBatchSinkConfig.java new file mode 100644 index 0000000..c685674 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/JMSBatchSinkConfig.java @@ -0,0 +1,114 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.jms.common.JMSConfig; +import io.cdap.plugin.jms.common.JMSMessageType; + +import java.io.IOException; +import java.io.Serializable; +import javax.annotation.Nullable; + +/** + * Holds the necessary configurations for the JMS source plugin + */ +public class JMSBatchSinkConfig extends JMSConfig implements Serializable { + + // Params + public static final String NAME_DESTINATION = "destinationName"; + public static final String NAME_MESSAGE_TYPE = "messageType"; + public static final String NAME_OUTPUT_SCHEMA = "schema"; + + public static final String DESC_DESTINATION = "Name of the destination Queue/Topic. If the given Queue/Topic name" + + "is not resolved, a new Queue/Topic with the given name will get created."; + public static final String DESC_MESSAGE_TYPE = "Supports the following message types: Message, Text, Bytes, Map, " + + "and Object."; + public static final String DESC_OUTPUT_SCHEMA = "Output schema."; + + @Name(NAME_DESTINATION) + @Description(DESC_DESTINATION) + @Macro + private String destinationName; + + @Name(NAME_MESSAGE_TYPE) + @Description(DESC_MESSAGE_TYPE) + @Nullable + @Macro + private String messageType; // default: Text + + @Name(NAME_OUTPUT_SCHEMA) + @Description(DESC_OUTPUT_SCHEMA) + @Nullable + @Macro + private String schema; + + public JMSBatchSinkConfig() { + super(""); + this.messageType = Strings.isNullOrEmpty(messageType) ? JMSMessageType.TEXT : messageType; + } + + public JMSBatchSinkConfig(String referenceName, String connectionFactory, String jmsUsername, + String jmsPassword, String providerUrl, String type, String jndiContextFactory, + String jndiUsername, String jndiPassword, String messageType, String destinationName) { + super(referenceName, connectionFactory, jmsUsername, jmsPassword, providerUrl, type, jndiContextFactory, + jndiUsername, jndiPassword); + this.destinationName = destinationName; + this.messageType = messageType; + } + + public String getDestinationName() { + return destinationName; + } + + public void validateParams(FailureCollector failureCollector) { + this.validateParams(failureCollector); + + if (Strings.isNullOrEmpty(destinationName) && !containsMacro(NAME_DESTINATION)) { + failureCollector + .addFailure("The destination topic/queue name must be provided!", "Provide your topic/queue name.") + .withConfigProperty(NAME_DESTINATION); + } + } + + public String getMessageType() { + if (!Strings.isNullOrEmpty(NAME_MESSAGE_TYPE) && !containsMacro(NAME_MESSAGE_TYPE)) { + return messageType; + } + return JMSMessageType.TEXT; + } + + /** + * @return {@link io.cdap.cdap.api.data.schema.Schema} of the dataset if one was given + * @throws IllegalArgumentException if the schema is not a valid JSON + */ + public Schema getSchema() { + if (!Strings.isNullOrEmpty(schema)) { + try { + return Schema.parseJson(schema); + } catch (IOException e) { + throw new IllegalArgumentException(String.format("Invalid schema : %s", e.getMessage()), e); + } + } + return null; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormat.java b/src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormat.java new file mode 100644 index 0000000..a3b69b1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormat.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.JobContext; +import org.apache.hadoop.mapreduce.OutputCommitter; +import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; + +/** + * Output format to write to JMS. + */ +public class JMSOutputFormat extends OutputFormat { + + @Override + public RecordWriter getRecordWriter(TaskAttemptContext context) { + return new JMSRecordWriter(context); + } + + @Override + public void checkOutputSpecs(JobContext jobContext) { + // no-op + } + + @Override + public OutputCommitter getOutputCommitter(TaskAttemptContext taskAttemptContext) { + return new OutputCommitter() { + @Override + public void setupJob(JobContext jobContext) { + // no-op + } + + @Override + public void setupTask(TaskAttemptContext taskAttemptContext) { + // no-op + } + + @Override + public boolean needsTaskCommit(TaskAttemptContext taskAttemptContext) { + return false; + } + + @Override + public void commitTask(TaskAttemptContext taskAttemptContext) { + // no-op + } + + @Override + public void abortTask(TaskAttemptContext taskAttemptContext) { + // no-op + } + }; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormatProvider.java b/src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormatProvider.java new file mode 100644 index 0000000..d433167 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/JMSOutputFormatProvider.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink; + +import com.google.common.collect.ImmutableMap; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.batch.OutputFormatProvider; + +import java.util.Map; + +/** + * JMS Output format provider. + */ +public class JMSOutputFormatProvider implements OutputFormatProvider { + + public static final String PROPERTY_CONFIG_JSON = "cdap.jms.sink.config"; + private static final Gson GSON = new GsonBuilder().create(); + private final Map conf; + + public JMSOutputFormatProvider(JMSBatchSinkConfig config) { + this.conf = new ImmutableMap.Builder() + .put(PROPERTY_CONFIG_JSON, GSON.toJson(config)) + .build(); + } + + @Override + public String getOutputFormatClassName() { + return JMSOutputFormat.class.getName(); + } + + @Override + public Map getOutputFormatConfiguration() { + return conf; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/JMSRecordWriter.java b/src/main/java/io/cdap/plugin/jms/sink/JMSRecordWriter.java new file mode 100644 index 0000000..5dabe3d --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/JMSRecordWriter.java @@ -0,0 +1,98 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSConnection; +import io.cdap.plugin.jms.sink.converters.SinkMessageConverterFacade; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.io.NullWritable; +import org.apache.hadoop.mapreduce.RecordWriter; +import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageProducer; +import javax.jms.Session; +import javax.naming.Context; + +/** + * Record writer to produce messages to a JMS Topic/Queue. + */ +public class JMSRecordWriter extends RecordWriter { + private static final Logger LOG = LoggerFactory.getLogger(JMSRecordWriter.class); + private static final Gson GSON = new GsonBuilder().create(); + + private final JMSBatchSinkConfig config; + private Connection connection; + private Session session; + private MessageProducer messageProducer; + private JMSConnection jmsConnection; + + public JMSRecordWriter(TaskAttemptContext context) { + Configuration config = context.getConfiguration(); + String configJson = config.get(JMSOutputFormatProvider.PROPERTY_CONFIG_JSON); + this.config = GSON.fromJson(configJson, JMSBatchSinkConfig.class); + this.jmsConnection = new JMSConnection(this.config); + establishConnection(); + } + + @Override + public void write(NullWritable key, StructuredRecord record) { + String messageType = config.getMessageType(); + Schema outputSchema = config.getSchema(); + Message message = null; + try { + message = SinkMessageConverterFacade.toJmsMessage(session, record, outputSchema, messageType); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + produceMessage(message); + } + + @Override + public void close(TaskAttemptContext taskAttemptContext) { + this.jmsConnection.stopConnection(this.connection); + this.jmsConnection.closeSession(this.session); + this.jmsConnection.closeConnection(this.connection); + } + + private void produceMessage(Message message) { + try { + messageProducer.send(message); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + + private void establishConnection() { + Context context = jmsConnection.getContext(); + ConnectionFactory factory = jmsConnection.getConnectionFactory(context); + connection = jmsConnection.createConnection(factory); + session = jmsConnection.createSession(connection); + Destination destination = jmsConnection.getSink(context, session); + messageProducer = jmsConnection.createProducer(session, destination); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverter.java b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverter.java new file mode 100644 index 0000000..f0b6c6c --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverter.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import javax.jms.BytesMessage; +import javax.jms.JMSException; + +/** + * A class with the functionality to convert StructuredRecords to BytesMessages. + */ +public class RecordToBytesMessageConverter { + + /** + * Converts an incoming {@link StructuredRecord} to a JMS {@link BytesMessage} + * + * @param bytesMessage the jms message to be populated with data + * @param record the incoming record + * @return a JMS bytes message + */ + public static BytesMessage toBytesMessage(BytesMessage bytesMessage, StructuredRecord record) { + try { + for (Schema.Field field : record.getSchema().getFields()) { + String fieldName = field.getName(); + Object value = record.get(fieldName); + + switch (field.getSchema().getType()) { + case INT: + bytesMessage.writeInt(cast(value, Integer.class)); + break; + case LONG: + bytesMessage.writeLong(cast(value, Long.class)); + break; + case DOUBLE: + bytesMessage.writeDouble(cast(value, Double.class)); + break; + case FLOAT: + bytesMessage.writeFloat(cast(value, Float.class)); + break; + case BOOLEAN: + bytesMessage.writeBoolean(cast(value, Boolean.class)); + break; + case BYTES: + bytesMessage.writeBytes(cast(value, byte[].class)); + break; + default: + bytesMessage.writeUTF(cast(value, String.class)); + } + } + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return bytesMessage; + } + + public static T cast(Object o, Class clazz) { + return o != null ? clazz.cast(o) : null; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverter.java b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverter.java new file mode 100644 index 0000000..6da8d4e --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverter.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import javax.jms.JMSException; +import javax.jms.MapMessage; + +/** + * A class with the functionality to convert StructuredRecords to MapMessages. + */ +public class RecordToMapMessageConverter { + + /** + * Converts an incoming {@link StructuredRecord} to a JMS {@link MapMessage} + * + * @param mapMessage the jms message to be populated with data + * @param record the incoming record + * @return a JMS map message + */ + public static MapMessage toMapMessage(MapMessage mapMessage, StructuredRecord record) { + try { + for (Schema.Field field : record.getSchema().getFields()) { + String fieldName = field.getName(); + Object value = record.get(fieldName); + + switch (field.getSchema().getType()) { + case INT: + mapMessage.setInt(fieldName, cast(value, Integer.class)); + break; + case LONG: + mapMessage.setLong(fieldName, cast(value, Long.class)); + break; + case DOUBLE: + mapMessage.setDouble(fieldName, cast(value, Double.class)); + break; + case FLOAT: + mapMessage.setFloat(fieldName, cast(value, Float.class)); + break; + case BOOLEAN: + mapMessage.setBoolean(fieldName, cast(value, Boolean.class)); + break; + case BYTES: + mapMessage.setBytes(fieldName, cast(value, byte[].class)); + break; + default: + mapMessage.setString(fieldName, cast(value, String.class)); + } + } + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return mapMessage; + } + + public static T cast(Object o, Class clazz) { + return o != null ? clazz.cast(o) : null; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverter.java b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverter.java new file mode 100644 index 0000000..90dcae2 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverter.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; + +import javax.jms.JMSException; +import javax.jms.Message; + +/** + * A class with the functionality to convert StructuredRecords to Messages. + */ +public class RecordToMessageConverter { + + /** + * Converts an incoming {@link StructuredRecord} to a JMS {@link Message} + * + * @param message the jms message to be populated with data + * @param record the incoming record + * @return a JMS message + */ + public static Message toMessage(Message message, StructuredRecord record) { + try { + for (Schema.Field field : record.getSchema().getFields()) { + String fieldName = field.getName(); + Object value = record.get(fieldName); + + switch (field.getSchema().getType()) { + case INT: + message.setIntProperty(fieldName, cast(value, Integer.class)); + break; + case LONG: + message.setLongProperty(fieldName, cast(value, Long.class)); + break; + case DOUBLE: + message.setDoubleProperty(fieldName, cast(value, Double.class)); + break; + case FLOAT: + message.setFloatProperty(fieldName, cast(value, Float.class)); + break; + case BOOLEAN: + message.setBooleanProperty(fieldName, cast(value, Boolean.class)); + break; + case BYTES: + message.setByteProperty(fieldName, cast(value, Byte.class)); + break; + default: + message.setStringProperty(fieldName, cast(value, String.class)); + } + } + + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return message; + } + + public static T cast(Object o, Class clazz) { + return o != null ? clazz.cast(o) : null; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverter.java b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverter.java new file mode 100644 index 0000000..2a1ce26 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverter.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.format.StructuredRecordStringConverter; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import javax.jms.JMSException; +import javax.jms.ObjectMessage; + +/** + * A class with the functionality to convert StructuredRecords to ObjectMessages. + */ +public class RecordToObjectMessageConverter { + + /** + * Converts an incoming {@link StructuredRecord} to a JMS {@link ObjectMessage} + * + * @param objectMessage the incoming record + * @param record the incoming record + * @return a JMS object message + */ + public static ObjectMessage toObjectMessage(ObjectMessage objectMessage, StructuredRecord record) { + byte[] body = null; + + try { + body = StructuredRecordStringConverter.toJsonString(record).getBytes(StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException("Failed to convert record to json!", e); + } + + try { + objectMessage.setObject(body); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return objectMessage; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverter.java b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverter.java new file mode 100644 index 0000000..036e657 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverter.java @@ -0,0 +1,90 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.format.StructuredRecordStringConverter; + +import java.io.IOException; +import javax.jms.JMSException; +import javax.jms.TextMessage; + +/** + * A class with the functionality to convert StructuredRecords to TextMessages. + */ +public class RecordToTextMessageConverter { + + /** + * Converts an incoming {@link StructuredRecord} to a JMS {@link TextMessage} + * + * @param textMessage the jms message to be populated with data + * @param record the incoming record + * @return a JMS text message + */ + public static TextMessage toTextMessage(TextMessage textMessage, StructuredRecord record) { + int numFields = record.getSchema().getFields().size(); + + if (numFields == 1) { + return withSingleField(textMessage, record); + } + return withMultipleFields(textMessage, record); + } + + /** + * Converts an incoming {@link StructuredRecord} with a single field to a JMS {@link TextMessage} where the text + * isn't wrapped in a json object. + * + * @param textMessage the jms message to be populated with data + * @param record the incoming record + * @return a JMS text message + */ + private static TextMessage withSingleField(TextMessage textMessage, StructuredRecord record) { + Schema.Field singleField = record.getSchema().getFields().get(0); + String body = record.get(singleField.getName()).toString(); + try { + textMessage.setText(body); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return textMessage; + } + + /** + * Converts an incoming {@link StructuredRecord} with multiple fields to a JMS {@link TextMessage} where the text + * is wrapped in a json object. + * + * @param textMessage the jms message to be populated with data + * @param record the incoming record + * @return a JMS text message + */ + private static TextMessage withMultipleFields(TextMessage textMessage, StructuredRecord record) { + String body; + + try { + body = StructuredRecordStringConverter.toJsonString(record); + } catch (IOException e) { + throw new RuntimeException("Failed to convert record to json!", e); + } + try { + textMessage.setText(body); + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + return textMessage; + } +} diff --git a/src/main/java/io/cdap/plugin/jms/sink/converters/SinkMessageConverterFacade.java b/src/main/java/io/cdap/plugin/jms/sink/converters/SinkMessageConverterFacade.java new file mode 100644 index 0000000..93f0ea1 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/sink/converters/SinkMessageConverterFacade.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageType; + +import javax.jms.BytesMessage; +import javax.jms.JMSException; +import javax.jms.MapMessage; +import javax.jms.Message; +import javax.jms.ObjectMessage; +import javax.jms.Session; +import javax.jms.TextMessage; + +/** + * A facade class that provides the functionality to convert different type of JMS messages to structured records. + */ +public class SinkMessageConverterFacade { + + /** + * + * @param session + * @param record + * @param outputSchema + * @param messageType + * @return + * @throws JMSException + */ + public static Message toJmsMessage(Session session, StructuredRecord record, Schema outputSchema, + String messageType) throws JMSException { + if (outputSchema == null) { + TextMessage textMessage = session.createTextMessage(); + return RecordToTextMessageConverter.toTextMessage(textMessage, record); + } else { + switch (messageType) { + case JMSMessageType.MAP: + MapMessage mapMessage = session.createMapMessage(); + return RecordToMapMessageConverter.toMapMessage(mapMessage, record); + + case JMSMessageType.BYTES: + BytesMessage bytesMessage = session.createBytesMessage(); + return RecordToBytesMessageConverter.toBytesMessage(bytesMessage, record); + + case JMSMessageType.MESSAGE: + Message message = session.createMessage(); + return RecordToMessageConverter.toMessage(message, record); + + case JMSMessageType.OBJECT: + ObjectMessage objectMessage = session.createObjectMessage(); + return RecordToObjectMessageConverter.toObjectMessage(objectMessage, record); + + case JMSMessageType.TEXT: + default: + TextMessage textMessage = session.createTextMessage(); + return RecordToTextMessageConverter.toTextMessage(textMessage, record); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/JMSReceiver.java b/src/main/java/io/cdap/plugin/jms/source/JMSReceiver.java new file mode 100644 index 0000000..664b9c5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/JMSReceiver.java @@ -0,0 +1,105 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.jms.common.JMSConnection; +import io.cdap.plugin.jms.source.converters.SourceMessageConverterFacade; +import org.apache.spark.storage.StorageLevel; +import org.apache.spark.streaming.receiver.Receiver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jms.Connection; +import javax.jms.ConnectionFactory; +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageConsumer; +import javax.jms.MessageListener; +import javax.jms.Queue; +import javax.jms.Session; +import javax.naming.Context; + +/** + * This class creates a customized message Receiver and implements the MessageListener interface. + */ +public class JMSReceiver extends Receiver implements MessageListener { + private static final Logger LOG = LoggerFactory.getLogger(JMSReceiver.class); + private JMSStreamingSourceConfig config; + private Connection connection; + private StorageLevel storageLevel; + private Session session; + private JMSConnection jmsConnection; + + public JMSReceiver(StorageLevel storageLevel, JMSStreamingSourceConfig config) { + super(storageLevel); + this.storageLevel = storageLevel; + this.config = config; + } + + @Override + public void onStart() { + this.jmsConnection = new JMSConnection(config); + Context context = jmsConnection.getContext(); + ConnectionFactory factory = jmsConnection.getConnectionFactory(context); + connection = jmsConnection.createConnection(factory); + + session = jmsConnection.createSession(connection); + Destination destination = jmsConnection.getSource(context); + MessageConsumer messageConsumer = jmsConnection.createConsumer(session, destination); + jmsConnection.setMessageListener(this, messageConsumer); + jmsConnection.startConnection(connection); + + // fetch the entire queue events + if (destination instanceof Queue) { + fetchEntireQueue(messageConsumer); + } + } + + @Override + public void onStop() { + this.jmsConnection.stopConnection(this.connection); + this.jmsConnection.closeSession(this.session); + this.jmsConnection.closeConnection(this.connection); + } + + @Override + public void onMessage(Message message) { + try { + store(SourceMessageConverterFacade.toStructuredRecord(message, this.config)); + } catch (Exception e) { + LOG.error("Message couldn't get stored in the Spark memory.", e); + throw new RuntimeException(e); + } + } + + private void fetchEntireQueue(MessageConsumer messageConsumer) { + while (true) { + try { + Message message = messageConsumer.receive(5000); + if (message != null) { + store(SourceMessageConverterFacade.toStructuredRecord(message, this.config)); + } else { + break; + } + } catch (JMSException e) { + throw new RuntimeException(String.format("%s: %s", e.getErrorCode(), e.getMessage())); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/JMSSourceUtils.java b/src/main/java/io/cdap/plugin/jms/source/JMSSourceUtils.java new file mode 100644 index 0000000..97f513e --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/JMSSourceUtils.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.etl.api.streaming.StreamingContext; +import org.apache.spark.storage.StorageLevel; +import org.apache.spark.streaming.api.java.JavaDStream; +import org.apache.spark.streaming.receiver.Receiver; + +/** + * Utils for the JMS source plugin. + */ +public class JMSSourceUtils { + + /** + * Creates a {@link JavaDStream} out of the {@link JMSReceiver} class. + * @param context the spark streaming context + * @param config the jms streaming source config + * @return the stream + */ + public static JavaDStream getJavaDStream(StreamingContext context, + JMSStreamingSourceConfig config) { + Receiver jmsReceiver = new JMSReceiver(StorageLevel.MEMORY_AND_DISK_SER_2(), config); + return context.getSparkStreamingContext().receiverStream(jmsReceiver); + } + +} diff --git a/src/main/java/io/cdap/plugin/jms/source/JMSStreamingSource.java b/src/main/java/io/cdap/plugin/jms/source/JMSStreamingSource.java new file mode 100644 index 0000000..8958fa5 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/JMSStreamingSource.java @@ -0,0 +1,76 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source; + +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.streaming.StreamingContext; +import io.cdap.cdap.etl.api.streaming.StreamingSource; +import io.cdap.cdap.etl.api.streaming.StreamingSourceContext; +import io.cdap.plugin.common.LineageRecorder; +import org.apache.spark.streaming.api.java.JavaDStream; + +import java.util.stream.Collectors; + +/** + * This class is a plugin that allows consuming messages from a specified JMS Queue/Topic and generate + * StructuredRecords out of them. + */ +@Plugin(type = StreamingSource.PLUGIN_TYPE) +@Name("JMS") +@Description("JMSSource") +public class JMSStreamingSource extends ReferenceStreamingSource { + + private JMSStreamingSourceConfig config; + + public JMSStreamingSource(JMSStreamingSourceConfig config) { + super(config); + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + super.configurePipeline(pipelineConfigurer); + config.validate(pipelineConfigurer.getStageConfigurer().getFailureCollector()); + pipelineConfigurer.getStageConfigurer().setOutputSchema(config.getSchema()); + } + + @Override + public void prepareRun(StreamingSourceContext context) throws Exception { + Schema schema = config.getSchema(); + context.registerLineage(config.referenceName, schema); + + if (schema.getFields() != null) { + LineageRecorder recorder = new LineageRecorder(context, config.referenceName); + recorder.recordRead("Read", "Read from jms", + schema.getFields().stream().map(Schema.Field::getName).collect(Collectors.toList())); + } + } + + @Override + public JavaDStream getStream(StreamingContext context) throws Exception { + FailureCollector collector = context.getFailureCollector(); + config.validate(collector); + collector.getOrThrowException(); + return JMSSourceUtils.getJavaDStream(context, config); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/JMSStreamingSourceConfig.java b/src/main/java/io/cdap/plugin/jms/source/JMSStreamingSourceConfig.java new file mode 100644 index 0000000..6aee4cc --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/JMSStreamingSourceConfig.java @@ -0,0 +1,225 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Macro; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.plugin.jms.common.JMSConfig; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.common.SchemaValidationUtils; + +import java.io.IOException; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nullable; + +/** + * Configs for {@link JMSStreamingSource}. + */ +public class JMSStreamingSourceConfig extends JMSConfig implements Serializable { + public static final String NAME_SOURCE = "sourceName"; + public static final String NAME_SCHEMA = "schema"; + public static final String NAME_MESSAGE_HEADER = "messageHeader"; + public static final String NAME_MESSAGE_PROPERTIES = "messageProperties"; + public static final String NAME_MESSAGE_TYPE = "messageType"; + + + @Name(NAME_SOURCE) + @Description("Name of the source Queue/Topic. The Queue/Topic with the given name, should exist in order to read " + + "messages from.") + @Macro + private String sourceName; + + @Name(NAME_MESSAGE_HEADER) + @Description("If true, message header is also consumed. Otherwise, it is not.") + @Nullable + @Macro + private String messageHeader; + + @Name(NAME_MESSAGE_PROPERTIES) + @Description("If true, message properties are also consumed. Otherwise, they are not.") + @Nullable + @Macro + private String messageProperties; + + @Name(NAME_MESSAGE_TYPE) + @Description("Supports the following message types: Message, Text, Bytes, Map, Object.") + @Nullable + @Macro + private String messageType; // default: Text + + @Name(NAME_SCHEMA) + @Description("Specifies the schema of the records outputted from this plugin.") + @Macro + private String schema; + + public JMSStreamingSourceConfig() { + super(""); + this.messageHeader = Strings.isNullOrEmpty(messageHeader) ? "true" : messageHeader; + this.messageType = Strings.isNullOrEmpty(messageType) ? JMSMessageType.TEXT : messageType; + } + + @VisibleForTesting + public JMSStreamingSourceConfig(String referenceName, String connectionFactory, String jmsUsername, + String jmsPassword, String providerUrl, String type, String jndiContextFactory, + String jndiUsername, String jndiPassword, String messageHeader, + String messageProperties, String messageType, String sourceName, String schema) { + super(referenceName, connectionFactory, jmsUsername, jmsPassword, providerUrl, type, jndiContextFactory, + jndiUsername, jndiPassword); + this.sourceName = sourceName; + this.messageHeader = messageHeader; + this.messageProperties = messageProperties; + this.messageType = messageType; + this.schema = schema; + } + + public void validate(FailureCollector failureCollector) { + this.validateParams(failureCollector); + + if (Strings.isNullOrEmpty(messageType) && !containsMacro(NAME_MESSAGE_TYPE)) { + failureCollector + .addFailure("The source topic/queue name must be provided!", "Provide your topic/queue name.") + .withConfigProperty(NAME_MESSAGE_TYPE); + } + + if (Strings.isNullOrEmpty(sourceName) && !containsMacro(NAME_SOURCE)) { + failureCollector + .addFailure("The source topic/queue name must be provided!", "Provide your topic/queue name.") + .withConfigProperty(NAME_SOURCE); + } + + if (!containsMacro(NAME_SCHEMA)) { + Schema schema = getSchema(); + + SchemaValidationUtils.validateIfAnyNotSupportedRootFieldExists(schema, failureCollector); + + if (getMessageHeader()) { + SchemaValidationUtils.validateHeaderSchema(schema, failureCollector); + } + + if (getMessageProperties()) { + SchemaValidationUtils.validatePropertiesSchema(schema, failureCollector); + } + + switch (messageType) { + case JMSMessageType.TEXT: + SchemaValidationUtils.validateIfBodyNotInSchema(schema, failureCollector); + SchemaValidationUtils.validateTextMessageSchema(schema, failureCollector); + break; + + case JMSMessageType.OBJECT: + SchemaValidationUtils.validateIfBodyNotInSchema(schema, failureCollector); + SchemaValidationUtils.validateObjectMessageSchema(schema, failureCollector); + break; + + case JMSMessageType.BYTES: + SchemaValidationUtils.validateIfBodyNotInSchema(schema, failureCollector); + SchemaValidationUtils.validateBytesMessageSchema(schema, failureCollector); + break; + + case JMSMessageType.MAP: + SchemaValidationUtils.validateIfBodyNotInSchema(schema, failureCollector); + SchemaValidationUtils.validateMapMessageSchema(schema, failureCollector); + break; + + case JMSMessageType.MESSAGE: + SchemaValidationUtils.validateMessageSchema(schema, failureCollector); + } + } + } + + public String getSourceName() { + return sourceName; + } + + /** + * @return {@link io.cdap.cdap.api.data.schema.Schema} of the dataset if one was given + * @throws IllegalArgumentException if the schema is not a valid JSON + */ + public Schema getSchema() { + + if (!Strings.isNullOrEmpty(schema) && !containsMacro(schema)) { + try { + return Schema.parseJson(schema); + } catch (IOException e) { + throw new IllegalArgumentException(String.format("Invalid schema : %s.", e.getMessage()), e); + } + } + + List fields = new ArrayList<>(); + + if (getMessageHeader()) { + fields.add(JMSMessageHeader.getMessageHeaderField()); + } + + if (getMessageProperties()) { + fields.add(Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING))); + } + + switch (messageType) { + case JMSMessageType.OBJECT: + fields.add(Schema.Field.of(JMSMessageParts.BODY, Schema.arrayOf(Schema.of(Schema.Type.BYTES)))); + return Schema.recordOf("record", fields); + + case JMSMessageType.MESSAGE: + if (!getMessageProperties() && !getMessageHeader()) { + SchemaValidationUtils.tell(null, SchemaValidationUtils.HEADER_AND_PROPERTIES_MISSING_IN_MESSAGE_ERROR, + SchemaValidationUtils.HEADER_AND_PROPERTIES_MISSING_IN_MESSAGE_ACTION); + } + return Schema.recordOf("record", fields); + + case JMSMessageType.MAP: + case JMSMessageType.TEXT: + case JMSMessageType.BYTES: + fields.add(Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING))); + return Schema.recordOf("record", fields); + default: + return Schema.recordOf("record", fields); + } + } + + @Nullable + public String getMessageType() { + return messageType; + } + + public boolean getMessageHeader() { + return this.messageHeader.equalsIgnoreCase("true"); + } + + public boolean getMessageProperties() { + return this.messageProperties.equalsIgnoreCase("true"); + } + + public List getDataFields(Schema schema, String skipFieldName) { + return schema + .getFields() + .stream() + .filter(field -> !JMSMessageHeader.getJMSMessageHeaderNames().contains(field.getName())) + .filter(field -> !field.getName().equals(skipFieldName)) + .collect(Collectors.toList()); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/ReferenceStreamingSource.java b/src/main/java/io/cdap/plugin/jms/source/ReferenceStreamingSource.java new file mode 100644 index 0000000..ddc7bf0 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/ReferenceStreamingSource.java @@ -0,0 +1,48 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source; + +import io.cdap.cdap.api.dataset.DatasetProperties; +import io.cdap.cdap.etl.api.FailureCollector; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.streaming.StreamingSource; +import io.cdap.plugin.common.Constants; +import io.cdap.plugin.common.IdUtils; +import io.cdap.plugin.common.ReferencePluginConfig; + +/** + * Base streaming source that adds an External Dataset for a reference name, and performs a single getDataset() + * call to make sure CDAP records that it was accessed. + * + * @param type of object read by the source. + */ +public abstract class ReferenceStreamingSource extends StreamingSource { + private final ReferencePluginConfig conf; + + public ReferenceStreamingSource(ReferencePluginConfig conf) { + this.conf = conf; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) throws IllegalArgumentException { + super.configurePipeline(pipelineConfigurer); + FailureCollector collector = pipelineConfigurer.getStageConfigurer().getFailureCollector(); + IdUtils.validateReferenceName(conf.referenceName, collector); + collector.getOrThrowException(); + pipelineConfigurer.createDataset(conf.referenceName, Constants.EXTERNAL_DATASET_TYPE, DatasetProperties.EMPTY); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverter.java b/src/main/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverter.java new file mode 100644 index 0000000..0c08ff8 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverter.java @@ -0,0 +1,211 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import com.google.gson.Gson; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageProperties; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; + +import java.io.ByteArrayOutputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import javax.jms.BytesMessage; +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.MessageEOFException; + +/** + * A class with the functionality to convert BytesMessages to StructuredRecords. + */ +public class BytesMessageToRecordConverter { + + /** + * Converts a {@link BytesMessage} to a {@link StructuredRecord} + * + * @param message the incoming JMS bytes message + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @return the structured record built out of the JMS bytes message fields + * @throws JMSException in case the method fails to read fields from the JMS message + */ + public static StructuredRecord bytesMessageToRecord(Message message, JMSStreamingSourceConfig config) + throws JMSException { + Schema schema = config.getSchema(); + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); + + if (config.getMessageHeader()) { + JMSMessageHeader.populateHeader(config.getSchema(), recordBuilder, message); + } + if (config.getMessageProperties()) { + JMSMessageProperties.populateProperties(schema, recordBuilder, message, JMSMessageType.MESSAGE); + } + + Schema.Type bodyFieldType = schema.getField(JMSMessageParts.BODY).getSchema().getType(); + + if (bodyFieldType.equals(Schema.Type.STRING)) { + byteMessageToRecordForStringBody(message, recordBuilder); + } else if (bodyFieldType.equals(Schema.Type.RECORD)) { + byteMessageToRecordForRecordBody(message, schema, recordBuilder, config); + } + return recordBuilder.build(); + } + + /** + * Converts a {@link BytesMessage} to a {@link StructuredRecord} when body is of data type string + * + * @param message the incoming JMS bytes message + * @param builder the {@link StructuredRecord.Builder} to enrich with body + * @throws JMSException in case the method fails to read fields from the JMS message + */ + private static void byteMessageToRecordForStringBody(Message message, StructuredRecord.Builder builder) + throws JMSException { + Map body = new LinkedHashMap<>(); + + // handle text data + try { + body.put("string_body", ((BytesMessage) message).readUTF()); + } catch (MessageEOFException e) { /* do nothing */ } + +// try { +// body.put("char_body", ((BytesMessage) message).readChar()); +// } catch (MessageEOFException e) { /* do nothing */ } + + // handle numerical data + try { + body.put("double_body", ((BytesMessage) message).readDouble()); + } catch (MessageEOFException e) { /* do nothing */ } + + try { + body.put("float_body", ((BytesMessage) message).readFloat()); + } catch (MessageEOFException e) { /* do nothing */ } + + try { + body.put("int_body", ((BytesMessage) message).readInt()); + } catch (MessageEOFException e) { /* do nothing */ } + + try { + body.put("long_body", ((BytesMessage) message).readLong()); + } catch (MessageEOFException e) { /* do nothing */ } + +// try { +// body.put("short_body", ((BytesMessage) message).readShort()); +// } catch (MessageEOFException e) { /* do nothing */ } + +// try { +// body.put("unsigned_short_body", ((BytesMessage) message).readUnsignedShort()); +// } catch (MessageEOFException e) { /* do nothing */ } + + // other + try { + body.put("boolean_body", ((BytesMessage) message).readBoolean()); + } catch (MessageEOFException e) { /* do nothing */ } + + try { + body.put("byte_body", ((BytesMessage) message).readByte()); + } catch (MessageEOFException e) { /* do nothing */ } + +// try { +// body.put("unsigned_byte_body", ((BytesMessage) message).readUnsignedByte()); +// } catch (MessageEOFException e) { /* do nothing */ } +// +// try { +// body.put("unsigned_byte_body", ((BytesMessage) message).readUnsignedByte()); +// } catch (MessageEOFException e) { /* do nothing */ } + + try { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8096]; + int currentByte; + while ((currentByte = ((BytesMessage) message).readBytes(buffer)) != -1) { + byteArrayOutputStream.write(buffer, 0, currentByte); + } + + body.put("bytes_body", byteArrayOutputStream.toByteArray()); + } catch (MessageEOFException e) { /* do nothing */ } + + builder.set(JMSMessageParts.BODY, new Gson().toJson(body)); + } + + /** + * Converts a {@link BytesMessage} to a {@link StructuredRecord} when body is of data type record + * + * @param message the incoming JMS bytes message + * @param schema the record schema + * @param builder the {@link StructuredRecord.Builder} to enrich with body + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @throws JMSException in case the method fails to read fields from the JMS message + */ + private static void byteMessageToRecordForRecordBody( + Message message, Schema schema, StructuredRecord.Builder builder, JMSStreamingSourceConfig config + ) throws JMSException { + Schema bodySchema = schema.getField(JMSMessageParts.BODY).getSchema(); + StructuredRecord.Builder bodyRecordBuilder = StructuredRecord.builder(bodySchema); + + for (Schema.Field field : bodySchema.getFields()) { + Schema.Type type = field.getSchema().getType(); + String name = field.getName(); + + if (type.equals(Schema.Type.UNION)) { + type = field.getSchema().getUnionSchema(0).getType(); + } + + switch (type) { + case BOOLEAN: + bodyRecordBuilder.set(name, ((BytesMessage) message).readBoolean()); + break; + case INT: + bodyRecordBuilder.set(name, ((BytesMessage) message).readInt()); + break; + case LONG: + bodyRecordBuilder.set(name, ((BytesMessage) message).readLong()); + break; + case FLOAT: + bodyRecordBuilder.set(name, ((BytesMessage) message).readFloat()); + break; + case DOUBLE: + bodyRecordBuilder.set(name, ((BytesMessage) message).readDouble()); + break; + case BYTES: + bodyRecordBuilder.set(name, ((BytesMessage) message).readByte()); + break; + case STRING: + bodyRecordBuilder.set(name, ((BytesMessage) message).readUTF()); + break; + case ARRAY: // byte array only + Schema.Type itemType = field.getSchema().getComponentSchema().getType(); + if (itemType.equals(Schema.Type.UNION)) { + itemType = field.getSchema().getComponentSchema().getUnionSchema(0).getType(); + } + if (itemType.equals(Schema.Type.BYTES)) { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8096]; + int currentByte; + while ((currentByte = ((BytesMessage) message).readBytes(buffer)) != -1) { + byteArrayOutputStream.write(buffer, 0, currentByte); + } + bodyRecordBuilder.set(name, byteArrayOutputStream.toByteArray()); + } + break; + } + } + builder.set(JMSMessageParts.BODY, bodyRecordBuilder.build()); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverter.java b/src/main/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverter.java new file mode 100644 index 0000000..da49373 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverter.java @@ -0,0 +1,147 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import com.google.gson.Gson; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageProperties; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; + +import java.util.Enumeration; +import java.util.LinkedHashMap; +import java.util.Map; +import javax.jms.JMSException; +import javax.jms.MapMessage; +import javax.jms.Message; + +/** + * A class with the functionality to convert MapMessages to StructuredRecords. + */ +public class MapMessageToRecordConverter { + + /** + * Creates a {@link StructuredRecord} from a JMS {@link MapMessage} + * + * @param message the incoming JMS {@link MapMessage} + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @return the {@link StructuredRecord} built out of the JMS {@link MapMessage} fields + * @throws JMSException in case the method fails to read fields from the JMS message + */ + public static StructuredRecord mapMessageToRecord(Message message, JMSStreamingSourceConfig config) + throws JMSException { + + Schema schema = config.getSchema(); + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); + + if (config.getMessageHeader()) { + JMSMessageHeader.populateHeader(schema, recordBuilder, message); + } + if (config.getMessageProperties()) { + JMSMessageProperties.populateProperties(schema, recordBuilder, message, JMSMessageType.MESSAGE); + } + + Schema.Type bodyFieldType = schema.getField(JMSMessageParts.BODY).getSchema().getType(); + + if (bodyFieldType.equals(Schema.Type.STRING)) { + withRecordBody(message, recordBuilder); + } else if (bodyFieldType.equals(Schema.Type.RECORD)) { + withRecordBody(message, schema, recordBuilder, config); + } + + return recordBuilder.build(); + } + + /** + * Creates a {@link StructuredRecord} from a JMS {@link MapMessage} when body is of data type string + * + * @param message the incoming JMS {@link MapMessage} + * @param builder the {@link StructuredRecord.Builder} to enrich with body + * @throws JMSException + */ + private static void withRecordBody(Message message, StructuredRecord.Builder builder) + throws JMSException { + Map body = new LinkedHashMap<>(); + Enumeration names = ((MapMessage) message).getMapNames(); + while (names.hasMoreElements()) { + String key = names.nextElement(); + body.put(key, ((MapMessage) message).getObject(key)); + } + builder.set(JMSMessageParts.BODY, new Gson().toJson(body)); + } + + /** + * Converts a {@link MapMessage} to a {@link StructuredRecord} when body is of data type record + * + * @param message the incoming JMS map message + * @param builder the {@link StructuredRecord.Builder} to enrich with body + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @throws JMSException + */ + private static void withRecordBody(Message message, Schema schema, StructuredRecord.Builder builder, + JMSStreamingSourceConfig config) + throws JMSException { + Schema bodySchema = schema.getField(JMSMessageParts.BODY).getSchema(); + StructuredRecord.Builder bodyRecordBuilder = StructuredRecord.builder(bodySchema); + + for (Schema.Field field : bodySchema.getFields()) { + Schema.Type type = field.getSchema().getType(); + String name = field.getName(); + + if (type.equals(Schema.Type.UNION)) { + type = field.getSchema().getUnionSchema(0).getType(); + } + + switch (type) { + case BOOLEAN: + bodyRecordBuilder.set(name, ((MapMessage) message).getBoolean(name)); + break; + case INT: + bodyRecordBuilder.set(name, ((MapMessage) message).getInt(name)); + break; + case LONG: + bodyRecordBuilder.set(name, ((MapMessage) message).getLong(name)); + break; + case FLOAT: + bodyRecordBuilder.set(name, ((MapMessage) message).getFloat(name)); + break; + case DOUBLE: + bodyRecordBuilder.set(name, ((MapMessage) message).getDouble(name)); + break; + case BYTES: + bodyRecordBuilder.set(name, ((MapMessage) message).getByte(name)); + break; + case STRING: + bodyRecordBuilder.set(name, ((MapMessage) message).getString(name)); + break; + case ARRAY: // byte array only + Schema.Type itemType = field.getSchema().getComponentSchema().getType(); + if (itemType.equals(Schema.Type.UNION)) { + itemType = field.getSchema().getComponentSchema().getUnionSchema(0).getType(); + } + if (itemType.equals(Schema.Type.BYTES)) { + bodyRecordBuilder.set(name, ((MapMessage) message).getBytes(name)); + } + break; + } + } + builder.set(JMSMessageParts.BODY, bodyRecordBuilder.build()); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverter.java b/src/main/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverter.java new file mode 100644 index 0000000..8b4837f --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverter.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageProperties; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; + +import javax.jms.JMSException; +import javax.jms.Message; + +/** + * A class with the functionality to convert Messages to StructuredRecords. + */ +public class MessageToRecordConverter { + + /** + * Creates a {@link StructuredRecord} from a JMS {@link Message} + * + * @param message the incoming JMS {@link Message} + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @return the {@link StructuredRecord} built out of the JMS {@link Message} fields + * @throws JMSException in case the method fails to read fields from the JMS message + */ + public static StructuredRecord messageToRecord(Message message, JMSStreamingSourceConfig config) + throws JMSException { + Schema schema = config.getSchema(); + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); + + if (config.getMessageHeader()) { + JMSMessageHeader.populateHeader(config.getSchema(), recordBuilder, message); + } + if (config.getMessageProperties()) { + JMSMessageProperties.populateProperties(schema, recordBuilder, message, JMSMessageType.MESSAGE); + } + return recordBuilder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverter.java b/src/main/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverter.java new file mode 100644 index 0000000..871c7e9 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverter.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageProperties; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import org.apache.commons.lang.SerializationUtils; + +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.ObjectMessage; + +/** + * A class with the functionality to convert ObjectMessages to StructuredRecords. + */ +public class ObjectMessageToRecordConverter { + + /** + * Creates a {@link StructuredRecord} from a JMS {@link ObjectMessage}. + * + * @param message the incoming JMS {@link ObjectMessage} + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @return the {@link StructuredRecord} built out of the JMS {@link ObjectMessage} fields + * @throws JMSException in case the method fails to read fields from the JMS message + */ + public static StructuredRecord objectMessageToRecord(Message message, JMSStreamingSourceConfig config) + throws JMSException { + Schema schema = config.getSchema(); + + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(schema); + + if (config.getMessageHeader()) { + JMSMessageHeader.populateHeader(config.getSchema(), recordBuilder, message); + } + if (config.getMessageProperties()) { + JMSMessageProperties.populateProperties(schema, recordBuilder, message, JMSMessageType.MESSAGE); + } + + byte[] body = SerializationUtils.serialize(((ObjectMessage) message).getObject()); + recordBuilder.set(JMSMessageParts.BODY, body); + return recordBuilder.build(); + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/converters/SourceMessageConverterFacade.java b/src/main/java/io/cdap/plugin/jms/source/converters/SourceMessageConverterFacade.java new file mode 100644 index 0000000..38abcce --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/converters/SourceMessageConverterFacade.java @@ -0,0 +1,56 @@ +package io.cdap.plugin.jms.source.converters; + +import com.google.common.base.Strings; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; + +import javax.jms.BytesMessage; +import javax.jms.JMSException; +import javax.jms.MapMessage; +import javax.jms.Message; +import javax.jms.ObjectMessage; +import javax.jms.TextMessage; + +/** + * A facade class that provides a single method to convert JMS messages to structured records. + */ +public class SourceMessageConverterFacade { + + /** + * Creates a {@link StructuredRecord} from a JMS message. + * + * @param message the incoming JMS message + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @return the {@link StructuredRecord} built out of the JMS message fields + * @throws JMSException in case the method fails to read fields from the JMS message + * @throws IllegalArgumentException in case the user provides a non-supported message type + */ + public static StructuredRecord toStructuredRecord(Message message, JMSStreamingSourceConfig config) + throws JMSException, IllegalArgumentException { + String messageType; + if (!Strings.isNullOrEmpty(config.getMessageType())) { + messageType = config.getMessageType(); + } else { + throw new RuntimeException("Message type should not be null."); + } + + if (message instanceof BytesMessage && messageType.equals(JMSMessageType.BYTES)) { + return BytesMessageToRecordConverter.bytesMessageToRecord(message, config); + } + if (message instanceof MapMessage && messageType.equals(JMSMessageType.MAP)) { + return MapMessageToRecordConverter.mapMessageToRecord(message, config); + } + if (message instanceof ObjectMessage && messageType.equals(JMSMessageType.OBJECT)) { + return ObjectMessageToRecordConverter.objectMessageToRecord(message, config); + } + if (message instanceof Message && messageType.equals(JMSMessageType.MESSAGE)) { + return ObjectMessageToRecordConverter.objectMessageToRecord(message, config); + } + if (message instanceof TextMessage && messageType.equals(JMSMessageType.TEXT)) { + return TextMessageToRecordConverter.textMessageToRecord(message, config); + } else { + throw new IllegalArgumentException("Message type should be one of Message, Text, Bytes, Map, or Object"); + } + } +} diff --git a/src/main/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverter.java b/src/main/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverter.java new file mode 100644 index 0000000..e10bf17 --- /dev/null +++ b/src/main/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverter.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageProperties; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; + +import javax.jms.JMSException; +import javax.jms.Message; +import javax.jms.TextMessage; + +/** + * A class with the functionality to convert TextMessages to StructuredRecords. + */ +public class TextMessageToRecordConverter { + + /** + * Creates a {@link StructuredRecord} from a JMS {@link TextMessage} + * + * @param message the incoming JMS {@link TextMessage} + * @param config the {@link JMSStreamingSourceConfig} with all user provided property values + * @return the {@link StructuredRecord} built out of the JMS {@link TextMessage} fields + * @throws JMSException in case the method fails to read fields from the JMS message + */ + public static StructuredRecord textMessageToRecord(Message message, JMSStreamingSourceConfig config) + throws JMSException { + StructuredRecord.Builder recordBuilder = StructuredRecord.builder(config.getSchema()); + + if (config.getMessageHeader()) { + JMSMessageHeader.populateHeader(config.getSchema(), recordBuilder, message); + } + if (config.getMessageProperties()) { + JMSMessageProperties.populateProperties(config.getSchema(), recordBuilder, message, config.getMessageType()); + } + + recordBuilder.set(JMSMessageParts.BODY, ((TextMessage) message).getText()); + return recordBuilder.build(); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/common/SchemaValidationUtilsTest.java b/src/test/java/io/cdap/plugin/jms/common/SchemaValidationUtilsTest.java new file mode 100644 index 0000000..a008201 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/common/SchemaValidationUtilsTest.java @@ -0,0 +1,261 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.common; + +import io.cdap.cdap.api.data.schema.Schema; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import static io.cdap.plugin.jms.common.SchemaValidationUtils.concatenate; + +/** + * Unit tests for schema validation. + */ +public class SchemaValidationUtilsTest { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + + @Test + public void validateIfSchemaIsNull_WithNullSchema_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.SCHEMA_IS_NULL_ERROR, + SchemaValidationUtils.SCHEMA_IS_NULL_ACTION + ) + ); + + Schema schema = null; + SchemaValidationUtils.validateIfSchemaIsNull(schema, null); + } + + @Test + public void validateIfSchemaIsNull_WithNotNullSchema_ShouldSucceed() { + Schema schema = Schema.recordOf("record", Schema.Field.of("test", Schema.of(Schema.Type.STRING))); + SchemaValidationUtils.validateIfSchemaIsNull(schema, null); + } + + @Test + public void validateIfAnyNotSupportedRootFieldExists_WithNotSupportedFields_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.NOT_SUPPORTED_ROOT_FIELDS_ERROR, + SchemaValidationUtils.NOT_SUPPORTED_ROOT_FIELDS_ACTION + ) + ); + + Schema schema = Schema + .recordOf("record", + Schema.Field.of(JMSMessageParts.HEADER, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)), + Schema.Field.of("other_field", Schema.of(Schema.Type.STRING)) + ); + + SchemaValidationUtils.validateIfAnyNotSupportedRootFieldExists(schema, null); + } + + @Test + public void validateIfAnyNotSupportedRootFieldExists_WithOnlySupportedFields_ShouldSucceed() { + Schema schema = Schema + .recordOf("record", + Schema.Field.of(JMSMessageParts.HEADER, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)) + ); + + SchemaValidationUtils.validateIfAnyNotSupportedRootFieldExists(schema, null); + } + + @Test + public void validateIfBodyNotInSchema_WithNoBody_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.BODY_NOT_IN_SCHEMA_ERROR, + SchemaValidationUtils.BODY_NOT_IN_SCHEMA_ACTION + ) + ); + + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.HEADER, Schema.of(Schema.Type.STRING))); + + SchemaValidationUtils.validateIfBodyNotInSchema(schema, null); + } + + @Test + public void validateIfBodyNotInSchema_WithBody_ShouldSucceed() { + Schema schema = Schema + .recordOf("record", + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.HEADER, Schema.of(Schema.Type.STRING)) + ); + SchemaValidationUtils.validateIfBodyNotInSchema(schema, null); + } + + @Test + public void validateTextMessageSchema_WithNonStringBody_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_TEXT_MESSAGE_ERROR, + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_TEXT_MESSAGE_ACTION + ) + ); + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.INT))); + + SchemaValidationUtils.validateTextMessageSchema(schema, null); + } + + @Test + public void validateMapMessageSchema_WithNonStringOrRecordBodyRootField_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_MAP_MESSAGE_ERROR, + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_MAP_MESSAGE_ACTION + ) + ); + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.INT))); + + SchemaValidationUtils.validateMapMessageSchema(schema, null); + } + + @Test + public void validateMessageSchema_WithNotSupportedRootFields_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.NOT_SUPPORTED_ROOT_FIELDS_IN_MESSAGE_ERROR, + SchemaValidationUtils.NOT_SUPPORTED_ROOT_FIELDS_IN_MESSAGE_ACTION + ) + ); + + Schema schema = Schema + .recordOf("record", + Schema.Field.of(JMSMessageParts.HEADER, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)), + Schema.Field.of("other-field", Schema.of(Schema.Type.STRING)) + ); + + SchemaValidationUtils.validateMessageSchema(schema, null); + } + + @Test + public void validateByteMessageSchema_WithNonStringOrRecordBodyRootField_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_BYTES_MESSAGE_ERROR, + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_BYTES_MESSAGE_ACTION + ) + ); + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.INT))); + + SchemaValidationUtils.validateBytesMessageSchema(schema, null); + } + + @Test + public void validateObjectMessageSchema_WithBodyNotArrayOfBytes_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_OBJECT_MESSAGE_ERROR, + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_IN_OBJECT_MESSAGE_ACTION + ) + ); + + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.arrayOf(Schema.of(Schema.Type.STRING))) + ); + + SchemaValidationUtils.validateObjectMessageSchema(schema, null); + } + + @Test + public void validateObjectMessageSchema_WithBodyArrayOfBytes_ShouldSucceed() { + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.arrayOf(Schema.of(Schema.Type.BYTES))) + ); + + SchemaValidationUtils.validateObjectMessageSchema(schema, null); + } + + @Test + public void validateHeaderSchema_WithHeaderNotRecord_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_FOR_HEADER_ERROR, + SchemaValidationUtils.WRONG_BODY_DATA_TYPE_FOR_HEADER_ACTION + ) + ); + + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.HEADER, Schema.of(Schema.Type.STRING)) + ); + + SchemaValidationUtils.validateHeaderSchema(schema, null); + } + + @Test + public void validateHeaderSchema_WithNonSupportedHeaderFields_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.NOT_SUPPORTED_FIELDS_IN_HEADER_RECORD_ERROR, + SchemaValidationUtils.NOT_SUPPORTED_FIELDS_IN_HEADER_RECORD_ACTION + ) + ); + + Schema schema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.HEADER, Schema.recordOf( + "record", + Schema.Field.of(JMSMessageHeader.MESSAGE_ID, Schema.of(Schema.Type.STRING)), + Schema.Field.of("other_field", Schema.of(Schema.Type.STRING)) + )) + ); + + SchemaValidationUtils.validateHeaderSchema(schema, null); + } + + @Test + public void validateProperties_WithNonStringOrRecordPropertiesRootField_ShouldThrowError() { + exceptionRule.expect(RuntimeException.class); + exceptionRule.expectMessage( + concatenate( + SchemaValidationUtils.WRONG_PROPERTIES_DATA_TYPE_ERROR, + SchemaValidationUtils.WRONG_PROPERTIES_DATA_TYPE_ACTION) + ); + + Schema schema = Schema + .recordOf("record", + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.INT)) + ); + + SchemaValidationUtils.validatePropertiesSchema(schema, null); + } +} + + diff --git a/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverterTest.java b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverterTest.java new file mode 100644 index 0000000..13aa298 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToBytesMessageConverterTest.java @@ -0,0 +1,59 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.apache.activemq.command.ActiveMQBytesMessage; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import javax.jms.BytesMessage; +import javax.jms.JMSException; + +import static org.mockito.Mockito.when; + +public class RecordToBytesMessageConverterTest { + + @Test + public void convertRecordToBytesMessage_Successfully() throws JMSException { + // actual + BytesMessage bytesMessage = new ActiveMQBytesMessage(); + + Schema schema = Schema.recordOf( + "record", + Schema.Field.of("FullName", Schema.of(Schema.Type.STRING)), + Schema.Field.of("Height", Schema.of(Schema.Type.DOUBLE)) + ); + + StructuredRecord record = StructuredRecord + .builder(schema) + .set("FullName", "Shaquille O'Neal") + .set("Height", 2.17) + .build(); + + BytesMessage actualMessage = RecordToBytesMessageConverter.toBytesMessage(bytesMessage, record); + BytesMessage mockedBytesMessage = Mockito.mock(actualMessage.getClass()); + when(mockedBytesMessage.readUTF()).thenReturn("Shaquille O'Neal"); + when(mockedBytesMessage.readDouble()).thenReturn(2.17); + + // assert + Assert.assertEquals(mockedBytesMessage.readUTF(), record.get("FullName")); + Assert.assertEquals((Double) mockedBytesMessage.readDouble(), record.get("Height")); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverterTest.java b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverterTest.java new file mode 100644 index 0000000..d568e88 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMapMessageConverterTest.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.apache.activemq.command.ActiveMQMapMessage; +import org.junit.Assert; +import org.junit.Test; + +import javax.jms.JMSException; +import javax.jms.MapMessage; + +public class RecordToMapMessageConverterTest { + + @Test + public void convertRecordToMapMessage_Successfully() throws JMSException { + // actual + MapMessage mapMessage = new ActiveMQMapMessage(); + Schema schema = Schema.recordOf( + "record", + Schema.Field.of("Name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("Surname", Schema.of(Schema.Type.STRING)), + Schema.Field.of("Age", Schema.of(Schema.Type.INT)) + ); + StructuredRecord record = StructuredRecord + .builder(schema) + .set("Name", "Robert") + .set("Surname", "Downey, Jr.") + .set("Age", 56) + .build(); + + MapMessage actualMessage = RecordToMapMessageConverter.toMapMessage(mapMessage, record); + + // asserts + Assert.assertEquals("Robert", actualMessage.getString("Name")); + Assert.assertEquals("Downey, Jr.", actualMessage.getString("Surname")); + Assert.assertEquals(56, actualMessage.getInt("Age")); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverterTest.java b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverterTest.java new file mode 100644 index 0000000..f4e7b2d --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToMessageConverterTest.java @@ -0,0 +1,55 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.apache.activemq.command.ActiveMQMapMessage; +import org.junit.Assert; +import org.junit.Test; + +import javax.jms.JMSException; +import javax.jms.Message; + +public class RecordToMessageConverterTest { + + @Test + public void convertRecordToMessage_Successfully() throws JMSException { + // actual + Message message = new ActiveMQMapMessage(); + + Schema schema = Schema.recordOf( + "record", + Schema.Field.of("IsWorkingStudent", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("HasWorkExperience", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("Position", Schema.of(Schema.Type.STRING)) + ); + StructuredRecord record = StructuredRecord + .builder(schema) + .set("IsWorkingStudent", true) + .set("HasWorkExperience", false) + .set("Position", "Frontend developer") + .build(); + + Message actualMessage = RecordToMessageConverter.toMessage(message, record); + + // Asserts + Assert.assertTrue(actualMessage.getBooleanProperty("IsWorkingStudent")); + Assert.assertFalse(actualMessage.getBooleanProperty("HasWorkExperience")); + Assert.assertEquals("Frontend developer", actualMessage.getStringProperty("Position")); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverterTest.java b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverterTest.java new file mode 100644 index 0000000..782b882 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToObjectMessageConverterTest.java @@ -0,0 +1,62 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.apache.activemq.command.ActiveMQObjectMessage; +import org.apache.commons.lang.SerializationUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.nio.charset.StandardCharsets; +import javax.jms.JMSException; +import javax.jms.ObjectMessage; + +public class RecordToObjectMessageConverterTest { + + @Test + public void convertRecordToObjectMessage_Successfully() throws JMSException { + // expected + String expectedStringBody = "{\"City\":\"Denver\",\"Population\":705000}"; + + // actual + ObjectMessage objectMessage = new ActiveMQObjectMessage(); + Schema schema = Schema.recordOf( + "record", + Schema.Field.of("City", Schema.of(Schema.Type.STRING)), + Schema.Field.of("Population", Schema.of(Schema.Type.INT)) + ); + + StructuredRecord record = StructuredRecord + .builder(schema) + .set("City", "Denver") + .set("Population", 705000) + .build(); + + ObjectMessage actualMessage = RecordToObjectMessageConverter.toObjectMessage(objectMessage, record); + byte[] actualBody = SerializationUtils.serialize(actualMessage.getObject()); + String actualStringBody = new String(actualBody, StandardCharsets.UTF_8); + + // assert + assertContains(expectedStringBody, actualStringBody); + } + + private static void assertContains(String expected, String actual) { + Assert.assertTrue(actual.contains(expected)); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverterTest.java b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverterTest.java new file mode 100644 index 0000000..6ca5e95 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/sink/converters/RecordToTextMessageConverterTest.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.sink.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import org.apache.activemq.command.ActiveMQTextMessage; +import org.junit.Assert; +import org.junit.Test; + +import javax.jms.JMSException; +import javax.jms.TextMessage; + +public class RecordToTextMessageConverterTest { + + @Test + public void convertRecordToTextMessage_WithSingleStringField_Successfully() throws JMSException { + // expected + String expectedBody = "Hello World!"; + + // actual + TextMessage textMessage = new ActiveMQTextMessage(); + Schema schema = Schema.recordOf("record", Schema.Field.of("text", Schema.of(Schema.Type.STRING))); + StructuredRecord record = StructuredRecord.builder(schema).set("text", "Hello World!").build(); + TextMessage actualMessage = RecordToTextMessageConverter.toTextMessage(textMessage, record); + String actualBody = actualMessage.getText(); + + // assert + Assert.assertEquals(expectedBody, actualBody); + } + + @Test + public void convertRecordToTextMessage_WithMultiplesFields_Successfully() throws JMSException { + // expected + String expectedBody = "{\"Name\":\"James\",\"Surname\":\"Bond\"}"; + + // actual + TextMessage textMessage = new ActiveMQTextMessage(); + Schema schema = Schema.recordOf( + "record", + Schema.Field.of("Name", Schema.of(Schema.Type.STRING)), + Schema.Field.of("Surname", Schema.of(Schema.Type.STRING)) + ); + + StructuredRecord record = StructuredRecord + .builder(schema) + .set("Name", "James") + .set("Surname", "Bond") + .build(); + + TextMessage actualMessage = RecordToTextMessageConverter.toTextMessage(textMessage, record); + String actualBody = actualMessage.getText(); + + // assert + Assert.assertEquals(expectedBody, actualBody); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverterTest.java b/src/test/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverterTest.java new file mode 100644 index 0000000..21f550f --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/converters/BytesMessageToRecordConverterTest.java @@ -0,0 +1,97 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import io.cdap.plugin.jms.source.utils.BytesMessageTestUtils; +import io.cdap.plugin.jms.source.utils.CommonTestUtils; +import org.junit.Test; + +import javax.jms.BytesMessage; +import javax.jms.JMSException; + +import static org.mockito.Mockito.mock; + +public class BytesMessageToRecordConverterTest { + + @Test + public void bytesMessageToRecord_WithNoHeaderNoPropertiesNoSchema_Successfully() throws JMSException { + BytesMessage bytesMessage = mock(BytesMessage.class); + BytesMessageTestUtils.mockBytesMessage(bytesMessage); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(false, false, JMSMessageType.BYTES, null); + + // expected + Schema expectedSchema = Schema.recordOf( + "record", + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)) + ); + + String expectedValue = BytesMessageTestUtils.getBytesMessageAsJsonString(); + StructuredRecord expectedRecord = StructuredRecord + .builder(expectedSchema) + .set(JMSMessageParts.BODY, expectedValue) + .build(); + + // actual + StructuredRecord actualRecord = BytesMessageToRecordConverter.bytesMessageToRecord(bytesMessage, config); + + // asserts + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } + + @Test + public void bytesMessageToRecord_WithNoHeaderNoPropertiesAndSchema_Successfully() throws JMSException { + BytesMessage bytesMessage = mock(BytesMessage.class); + BytesMessageTestUtils.mockBytesMessage(bytesMessage); + + // expected + Schema.Field bodyField = Schema.Field.of( + JMSMessageParts.BODY, + Schema.recordOf(JMSMessageParts.BODY, + Schema.Field.of("string_payload", Schema.of(Schema.Type.STRING)), + Schema.Field.of("int_payload", Schema.of(Schema.Type.INT)) + ) + ); + + Schema outputSchema = Schema.recordOf("record", bodyField); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(false, false, JMSMessageType.BYTES, + outputSchema.toString()); + + StructuredRecord expectedBodyRecord = StructuredRecord + .builder(bodyField.getSchema()) + .set("string_payload", "Hello!") + .set("int_payload", 1) + .build(); + + StructuredRecord expectedRecord = StructuredRecord + .builder(outputSchema) + .set(JMSMessageParts.BODY, expectedBodyRecord) + .build(); + + // actual + StructuredRecord actualRecord = BytesMessageToRecordConverter.bytesMessageToRecord(bytesMessage, config); + + // asserts + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverterTest.java b/src/test/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverterTest.java new file mode 100644 index 0000000..f8b050d --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/converters/MapMessageToRecordConverterTest.java @@ -0,0 +1,95 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import io.cdap.plugin.jms.source.utils.CommonTestUtils; +import io.cdap.plugin.jms.source.utils.HeaderTestUtils; +import io.cdap.plugin.jms.source.utils.MapMessageTestUtils; +import io.cdap.plugin.jms.source.utils.PropertiesTestUtils; +import org.apache.activemq.command.ActiveMQMapMessage; +import org.junit.Test; + +import javax.jms.JMSException; +import javax.jms.MapMessage; + +public class MapMessageToRecordConverterTest { + + + @Test + public void mapMessageToRecord_WithHeaderAndPropertiesAndNoSchema_Successfully() throws JMSException { + MapMessage mapMessage = new ActiveMQMapMessage(); + MapMessageTestUtils.addBodyValuesToMessage(mapMessage); + HeaderTestUtils.addHeaderValuesToMessage(mapMessage); + PropertiesTestUtils.addPropertiesToMessage(mapMessage); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(true, true, JMSMessageType.MAP, null); + + // expected + Schema expectedSchema = Schema.recordOf("record", JMSMessageHeader.getMessageHeaderField(), + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING))); + StructuredRecord.Builder expectedRecordBuilder = StructuredRecord.builder(expectedSchema); + HeaderTestUtils.addHeaderValuesToRecord(expectedRecordBuilder, expectedSchema); + PropertiesTestUtils.addPropertiesToRecord(expectedRecordBuilder, expectedSchema); + String expectedBody = MapMessageTestUtils.geBodyValuesAsJson(); + expectedRecordBuilder.set(JMSMessageParts.BODY, expectedBody); + StructuredRecord expectedRecord = expectedRecordBuilder.build(); + + // actual + StructuredRecord actualRecord = MapMessageToRecordConverter.mapMessageToRecord(mapMessage, config); + + // assert + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } + + @Test + public void mapMessageToRecord_WithHeaderAndPropertiesAndSchema_Successfully() throws JMSException { + MapMessage mapMessage = new ActiveMQMapMessage(); + MapMessageTestUtils.addBodyValuesToMessage(mapMessage); + HeaderTestUtils.addHeaderValuesToMessage(mapMessage); + PropertiesTestUtils.addPropertiesToMessage(mapMessage); + + Schema outputSchema = Schema.recordOf( + "record", + JMSMessageHeader.getMessageHeaderField(), + PropertiesTestUtils.getPropertiesField(), + MapMessageTestUtils.getBodyFields() + ); + + JMSStreamingSourceConfig config = CommonTestUtils + .getSourceConfig(true, true, JMSMessageType.MAP, outputSchema.toString()); + + // expected + StructuredRecord.Builder expectedRecordBuilder = StructuredRecord.builder(outputSchema); + HeaderTestUtils.addHeaderValuesToRecord(expectedRecordBuilder, outputSchema); + PropertiesTestUtils.addPropertiesToRecord(expectedRecordBuilder, outputSchema); + MapMessageTestUtils.addBodyValuesToRecord(expectedRecordBuilder, outputSchema); + StructuredRecord expectedRecord = expectedRecordBuilder.build(); + + // actual + StructuredRecord actualRecord = MapMessageToRecordConverter.mapMessageToRecord(mapMessage, config); + + // assert + CommonTestUtils.assertEqualsStructuredRecords(actualRecord, expectedRecord); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverterTest.java b/src/test/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverterTest.java new file mode 100644 index 0000000..51dca41 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/converters/MessageToRecordConverterTest.java @@ -0,0 +1,93 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import io.cdap.plugin.jms.source.utils.CommonTestUtils; +import io.cdap.plugin.jms.source.utils.HeaderTestUtils; +import io.cdap.plugin.jms.source.utils.PropertiesTestUtils; +import org.apache.activemq.command.ActiveMQBytesMessage; +import org.junit.Test; + +import java.util.Objects; +import javax.jms.Message; + +public class MessageToRecordConverterTest { + + @Test + public void messageToRecord_WithMetadataAndPropertiesAndNoSchema_Successfully() throws Exception { + Message message = new ActiveMQBytesMessage(); + HeaderTestUtils.addHeaderValuesToMessage(message); + PropertiesTestUtils.addPropertiesToMessage(message); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(true, true, JMSMessageType.MESSAGE, null); + + // expected + Schema expectedSchema = Schema.recordOf( + "record", + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING)), + JMSMessageHeader.getMessageHeaderField() + ); + StructuredRecord.Builder expectedRecordBuilder = StructuredRecord.builder(expectedSchema); + HeaderTestUtils.addHeaderValuesToRecord(expectedRecordBuilder, expectedSchema); + PropertiesTestUtils.addPropertiesToRecord(expectedRecordBuilder, expectedSchema); + StructuredRecord expectedRecord = expectedRecordBuilder.build(); + + // actual + StructuredRecord actualRecord = MessageToRecordConverter.messageToRecord(message, config); + + CommonTestUtils.assertEqualsStructuredRecords( + Objects.requireNonNull(expectedRecord.get(JMSMessageParts.HEADER)), + Objects.requireNonNull(actualRecord.get(JMSMessageParts.HEADER)) + ); + } + + @Test + public void messageToRecord_WithMetadataAndPropertiesAndSchema_Successfully() throws Exception { + Message message = new ActiveMQBytesMessage(); + HeaderTestUtils.addHeaderValuesToMessage(message); + PropertiesTestUtils.addPropertiesToMessage(message); + + Schema outputSchema = Schema.recordOf( + "record", + PropertiesTestUtils.getPropertiesField(), + JMSMessageHeader.getMessageHeaderField() + ); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(true, true, JMSMessageType.MESSAGE, + outputSchema.toString()); + + // expected + StructuredRecord.Builder expectedRecordBuilder = StructuredRecord.builder(outputSchema); + HeaderTestUtils.addHeaderValuesToRecord(expectedRecordBuilder, outputSchema); + PropertiesTestUtils.addPropertiesToRecord(expectedRecordBuilder, outputSchema); + StructuredRecord expectedRecord = expectedRecordBuilder.build(); + + // actual + StructuredRecord actualRecord = MessageToRecordConverter.messageToRecord(message, config); + + CommonTestUtils.assertEqualsStructuredRecords( + Objects.requireNonNull(expectedRecord.get(JMSMessageParts.HEADER)), + Objects.requireNonNull(actualRecord.get(JMSMessageParts.HEADER)) + ); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverterTest.java b/src/test/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverterTest.java new file mode 100644 index 0000000..997ef75 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/converters/ObjectMessageToRecordConverterTest.java @@ -0,0 +1,56 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import io.cdap.plugin.jms.source.utils.CommonTestUtils; +import io.cdap.plugin.jms.source.utils.DummyObject; +import org.apache.activemq.command.ActiveMQObjectMessage; +import org.apache.commons.lang.SerializationUtils; +import org.junit.Test; + +import javax.jms.JMSException; +import javax.jms.ObjectMessage; + +public class ObjectMessageToRecordConverterTest { + + @Test + public void objectMessageToRecord_WithNoHeaderNoPropertiesNoSchema_Successfully() throws JMSException { + DummyObject dummyObject = new DummyObject("Boeing", 777); + ObjectMessage objectMessage = new ActiveMQObjectMessage(); + objectMessage.setObject(dummyObject); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(false, false, JMSMessageType.OBJECT, null); + + // expected + Schema expectedSchema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.arrayOf(Schema.of(Schema.Type.BYTES)))); + StructuredRecord expectedRecord = StructuredRecord + .builder(expectedSchema) + .set(JMSMessageParts.BODY, SerializationUtils.serialize(dummyObject)).build(); + + // actual + StructuredRecord actualRecord = ObjectMessageToRecordConverter.objectMessageToRecord(objectMessage, config); + + // asserts + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverterTest.java b/src/test/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverterTest.java new file mode 100644 index 0000000..dc7116e --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/converters/TextMessageToRecordConverterTest.java @@ -0,0 +1,120 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.converters; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; +import io.cdap.plugin.jms.common.JMSMessageType; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import io.cdap.plugin.jms.source.utils.CommonTestUtils; +import io.cdap.plugin.jms.source.utils.HeaderTestUtils; +import io.cdap.plugin.jms.source.utils.PropertiesTestUtils; +import org.apache.activemq.command.ActiveMQTextMessage; +import org.junit.Test; + +import javax.jms.JMSException; +import javax.jms.TextMessage; + +public class TextMessageToRecordConverterTest { + + @Test + public void textMessageToRecord_WithNoHeaderNoPropertiesNoSchema_Successfully() throws JMSException { + TextMessage textMessage = new ActiveMQTextMessage(); + textMessage.setText("Hello World!"); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(false, false, JMSMessageType.TEXT, null); + + // expected + Schema expectedSchema = Schema + .recordOf("record", Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING))); + StructuredRecord expectedRecord = StructuredRecord + .builder(expectedSchema) + .set(JMSMessageParts.BODY, "Hello World!").build(); + + // actual + StructuredRecord actualRecord = TextMessageToRecordConverter.textMessageToRecord(textMessage, config); + + // asserts + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } + + @Test + public void textMessageToRecord_WithHeaderAndPropertiesAndNoSchema_Successfully() throws JMSException { + String bodyContent = "Hello World!"; + TextMessage textMessage = new ActiveMQTextMessage(); + textMessage.setText(bodyContent); + + HeaderTestUtils.addHeaderValuesToMessage(textMessage); + PropertiesTestUtils.addPropertiesToMessage(textMessage); + + JMSStreamingSourceConfig config = CommonTestUtils.getSourceConfig(true, true, JMSMessageType.TEXT, null); + + // expected + Schema expectedSchema = Schema.recordOf( + "record", + JMSMessageHeader.getMessageHeaderField(), + Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.of(Schema.Type.STRING)), + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)) + ); + StructuredRecord.Builder expectedRecordBuilder = StructuredRecord.builder(expectedSchema); + HeaderTestUtils.addHeaderValuesToRecord(expectedRecordBuilder, expectedSchema); + PropertiesTestUtils.addPropertiesToRecord(expectedRecordBuilder, expectedSchema); + expectedRecordBuilder.set(JMSMessageParts.BODY, bodyContent); + StructuredRecord expectedRecord = expectedRecordBuilder.build(); + + // actual + StructuredRecord actualRecord = TextMessageToRecordConverter.textMessageToRecord(textMessage, config); + + // asserts + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } + + @Test + public void textMessageToRecord_WithHeaderAndPropertiesAndSchema_Successfully() throws JMSException { + String bodyContent = "Hello World!"; + TextMessage textMessage = new ActiveMQTextMessage(); + textMessage.setText(bodyContent); + + HeaderTestUtils.addHeaderValuesToMessage(textMessage); + PropertiesTestUtils.addPropertiesToMessage(textMessage); + + Schema outputSchema = Schema.recordOf( + "record", + JMSMessageHeader.getMessageHeaderField(), + PropertiesTestUtils.getPropertiesField(), + Schema.Field.of(JMSMessageParts.BODY, Schema.of(Schema.Type.STRING)) + ); + + JMSStreamingSourceConfig config = CommonTestUtils + .getSourceConfig(true, true, JMSMessageType.TEXT, outputSchema.toString()); + + // expected + StructuredRecord.Builder expectedRecordBuilder = StructuredRecord.builder(outputSchema); + HeaderTestUtils.addHeaderValuesToRecord(expectedRecordBuilder, outputSchema); + PropertiesTestUtils.addPropertiesToRecord(expectedRecordBuilder, outputSchema); + expectedRecordBuilder.set(JMSMessageParts.BODY, bodyContent); + StructuredRecord expectedRecord = expectedRecordBuilder.build(); + + // actual + StructuredRecord actualRecord = TextMessageToRecordConverter.textMessageToRecord(textMessage, config); + + // asserts + CommonTestUtils.assertEqualsStructuredRecords(expectedRecord, actualRecord); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/utils/BytesMessageTestUtils.java b/src/test/java/io/cdap/plugin/jms/source/utils/BytesMessageTestUtils.java new file mode 100644 index 0000000..e07ba1d --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/utils/BytesMessageTestUtils.java @@ -0,0 +1,58 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.utils; + +import com.google.gson.Gson; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import javax.jms.BytesMessage; +import javax.jms.JMSException; + +import static org.mockito.Mockito.when; + +public class BytesMessageTestUtils { + + private static final Map DUMMY_DATA = + Collections.unmodifiableMap(new LinkedHashMap() {{ + put("string_body", "Hello!"); + put("double_body", 1D); + put("float_body", 1F); + put("int_body", 1); + put("long_body", 1L); + put("boolean_body", true); + put("byte_body", (byte) 1); + put("bytes_body", -1); + }}); + + public static String getBytesMessageAsJsonString() { + return new Gson().toJson(DUMMY_DATA).toString().replace("-1", "[]"); + } + + public static void mockBytesMessage(BytesMessage bytesMessage) throws JMSException { + when(bytesMessage.readBoolean()).thenReturn((boolean) DUMMY_DATA.get("boolean_body")); + when(bytesMessage.readByte()).thenReturn((byte) DUMMY_DATA.get("byte_body")); + when(bytesMessage.readInt()).thenReturn((int) DUMMY_DATA.get("int_body")); + when(bytesMessage.readLong()).thenReturn((long) DUMMY_DATA.get("long_body")); + when(bytesMessage.readFloat()).thenReturn((float) DUMMY_DATA.get("float_body")); + when(bytesMessage.readDouble()).thenReturn((double) DUMMY_DATA.get("double_body")); + when(bytesMessage.readUTF()).thenReturn((String) DUMMY_DATA.get("string_body")); + when(bytesMessage.readBytes(new byte[8096])).thenReturn((int) DUMMY_DATA.get("bytes_body")); + when(bytesMessage.getBodyLength()).thenReturn(1L); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/utils/CommonTestUtils.java b/src/test/java/io/cdap/plugin/jms/source/utils/CommonTestUtils.java new file mode 100644 index 0000000..c2480c3 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/utils/CommonTestUtils.java @@ -0,0 +1,86 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.utils; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.source.JMSStreamingSourceConfig; +import org.junit.Assert; + +import javax.jms.Destination; + +public class CommonTestUtils { + + public static JMSStreamingSourceConfig getSourceConfig(boolean messageHeader, boolean messageProperties, + String messageType, String schema) { + return new JMSStreamingSourceConfig("referenceName", "Connection Factory", "jms-username", "jms-password", + "tcp://0.0.0.0:61616", "Queue", "jndi-context-factory", "jndi-username", + "jndi-password", String.valueOf(messageHeader), + String.valueOf(messageProperties), messageType, "MyQueue", schema); + } + + public static Destination getDummyDestination() { + return new Destination() { + @Override + public String toString() { + return "Destination"; + } + }; + } + + public static void assertEqualsStructuredRecords(StructuredRecord expected, StructuredRecord actual) { + for (Schema.Field field : expected.getSchema().getFields()) { + Schema.Type type = field.getSchema().getType(); + + if (type.equals(Schema.Type.RECORD)) { + assertEqualsStructuredRecords(expected.get(field.getName()), actual.get(field.getName())); + } + + if (type.equals(Schema.Type.UNION)) { + type = field.getSchema().getUnionSchema(0).getType(); + } + + switch (type) { + case BOOLEAN: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case INT: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case LONG: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case FLOAT: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case DOUBLE: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case BYTES: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case STRING: + Assert.assertEquals(expected.get(field.getName()), actual.get(field.getName())); + break; + case ARRAY: + Assert.assertArrayEquals(expected.get(field.getName()), + actual.get(field.getName())); + break; + } + } + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/utils/DummyObject.java b/src/test/java/io/cdap/plugin/jms/source/utils/DummyObject.java new file mode 100644 index 0000000..d49fe76 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/utils/DummyObject.java @@ -0,0 +1,63 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.utils; + +import java.io.Serializable; +import java.util.Objects; + +public class DummyObject implements Serializable { + String dummyStr; + int dummyInt; + + public DummyObject(String dummyStr, int dummyInt) { + this.dummyStr = dummyStr; + this.dummyInt = dummyInt; + } + + public String getDummyStr() { + return dummyStr; + } + + public void setDummyStr(String dummyStr) { + this.dummyStr = dummyStr; + } + + public int getDummyInt() { + return dummyInt; + } + + public void setDummyInt(int dummyInt) { + this.dummyInt = dummyInt; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DummyObject that = (DummyObject) o; + return dummyInt == that.dummyInt && Objects.equals(dummyStr, that.dummyStr); + } + + @Override + public int hashCode() { + return Objects.hash(dummyStr, dummyInt); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/utils/HeaderTestUtils.java b/src/test/java/io/cdap/plugin/jms/source/utils/HeaderTestUtils.java new file mode 100644 index 0000000..670e811 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/utils/HeaderTestUtils.java @@ -0,0 +1,71 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.utils; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageHeader; +import io.cdap.plugin.jms.common.JMSMessageParts; + +import javax.jms.Destination; +import javax.jms.JMSException; +import javax.jms.Message; + +public class HeaderTestUtils { + + private static final String VAL_MESSAGE_ID = "JMSMessageID"; + private static final Long VAL_MESSAGE_TIMESTAMP = 1619096400L; + private static final String VAL_CORRELATION_ID = "JMSCorrelationID"; + private static final Destination VAL_REPLY_TO = null; + private static final Destination VAL_DESTINATION = VAL_REPLY_TO; + private static final String VAL_TYPE = "JMSType"; + private static final Integer VAL_DELIVERY_MODE = 1; + private static final Boolean VAL_REDELIVERED = false; + private static final Long VAL_EXPIRATION = VAL_MESSAGE_TIMESTAMP; + private static final Integer VAL_PRIORITY = 0; + + public static void addHeaderValuesToRecord(StructuredRecord.Builder builder, Schema schema) { + StructuredRecord.Builder headerBuilder = StructuredRecord + .builder(schema.getField(JMSMessageParts.HEADER).getSchema()); + + headerBuilder.set(JMSMessageHeader.MESSAGE_ID, "ID:" + VAL_MESSAGE_ID); + headerBuilder.set(JMSMessageHeader.MESSAGE_TIMESTAMP, VAL_MESSAGE_TIMESTAMP); + headerBuilder.set(JMSMessageHeader.CORRELATION_ID, VAL_CORRELATION_ID); + headerBuilder.set(JMSMessageHeader.REPLY_TO, VAL_REPLY_TO); + headerBuilder.set(JMSMessageHeader.DESTINATION, VAL_DESTINATION); + headerBuilder.set(JMSMessageHeader.TYPE, VAL_TYPE); + headerBuilder.set(JMSMessageHeader.DELIVERY_MODE, VAL_DELIVERY_MODE); + headerBuilder.set(JMSMessageHeader.REDELIVERED, VAL_REDELIVERED); + headerBuilder.set(JMSMessageHeader.EXPIRATION, VAL_EXPIRATION); + headerBuilder.set(JMSMessageHeader.PRIORITY, VAL_PRIORITY); + + builder.set(JMSMessageParts.HEADER, headerBuilder.build()); + } + + public static void addHeaderValuesToMessage(Message message) throws JMSException { + message.setJMSMessageID(VAL_MESSAGE_ID); + message.setJMSTimestamp(VAL_MESSAGE_TIMESTAMP); + message.setJMSCorrelationID(VAL_CORRELATION_ID); + message.setJMSReplyTo(VAL_REPLY_TO); + message.setJMSDestination(VAL_DESTINATION); + message.setJMSType(VAL_TYPE); + message.setJMSDeliveryMode(VAL_DELIVERY_MODE); + message.setJMSRedelivered(VAL_REDELIVERED); + message.setJMSExpiration(VAL_EXPIRATION); + message.setJMSPriority(VAL_PRIORITY); + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/utils/MapMessageTestUtils.java b/src/test/java/io/cdap/plugin/jms/source/utils/MapMessageTestUtils.java new file mode 100644 index 0000000..2fadb2f --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/utils/MapMessageTestUtils.java @@ -0,0 +1,79 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.utils; + +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageParts; + +import javax.jms.JMSException; +import javax.jms.MapMessage; + +public class MapMessageTestUtils { + + private static final Boolean BOOLEAN_VAL = true; + private static final Byte BYTE_VAL = (byte) 1; + private static final Integer INT_VAL = 1; + private static final Long LONG_VAL = 2L; + private static final Float FLOAT_VAL = 3F; + private static final Double DOUBLE_VAL = 4D; + private static final String STRING_VAL = "Hello World!"; + + public static Schema.Field getBodyFields() { + return Schema.Field.of( + JMSMessageParts.BODY, + Schema.recordOf( + JMSMessageParts.BODY, + Schema.Field.of("booleanVal", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("byteVal", Schema.of(Schema.Type.BYTES)), + Schema.Field.of("intVal", Schema.of(Schema.Type.INT)), + Schema.Field.of("longVal", Schema.of(Schema.Type.LONG)), + Schema.Field.of("floatVal", Schema.of(Schema.Type.FLOAT)), + Schema.Field.of("doubleVal", Schema.of(Schema.Type.DOUBLE)), + Schema.Field.of("stringVal", Schema.of(Schema.Type.STRING)) + ) + ); + } + + public static void addBodyValuesToMessage(MapMessage message) throws JMSException { + message.setBoolean("booleanVal", BOOLEAN_VAL); + message.setByte("byteVal", BYTE_VAL); + message.setInt("intVal", INT_VAL); + message.setLong("longVal", LONG_VAL); + message.setFloat("floatVal", FLOAT_VAL); + message.setDouble("doubleVal", DOUBLE_VAL); + message.setString("stringVal", STRING_VAL); + } + + public static void addBodyValuesToRecord(StructuredRecord.Builder builder, Schema schema) { + StructuredRecord bodyRecord = StructuredRecord.builder(schema.getField(JMSMessageParts.BODY).getSchema()) + .set("booleanVal", BOOLEAN_VAL) + .set("byteVal", BYTE_VAL) + .set("intVal", INT_VAL) + .set("longVal", LONG_VAL) + .set("floatVal", FLOAT_VAL) + .set("doubleVal", DOUBLE_VAL) + .set("stringVal", STRING_VAL) + .build(); + builder.set(JMSMessageParts.BODY, bodyRecord); + } + + public static String geBodyValuesAsJson() { + return "{\"intVal\":1,\"doubleVal\":4.0,\"byteVal\":1,\"stringVal\":\"Hello World!\",\"booleanVal\":true," + + "\"floatVal\":3.0,\"longVal\":2}"; + } +} diff --git a/src/test/java/io/cdap/plugin/jms/source/utils/PropertiesTestUtils.java b/src/test/java/io/cdap/plugin/jms/source/utils/PropertiesTestUtils.java new file mode 100644 index 0000000..8479ca3 --- /dev/null +++ b/src/test/java/io/cdap/plugin/jms/source/utils/PropertiesTestUtils.java @@ -0,0 +1,96 @@ +/* + * Copyright © 2021 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.jms.source.utils; + +import com.google.gson.Gson; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.jms.common.JMSMessageParts; + +import java.util.HashMap; +import javax.jms.JMSException; +import javax.jms.Message; + +public class PropertiesTestUtils { + + private static final Boolean BOOLEAN_PROPERTY = true; + private static final Byte BYTE_PROPERTY = (byte) 1; + private static final Integer INT_PROPERTY = 3; + private static final Long LONG_PROPERTY = 4L; + private static final Float FLOAT_PROPERTY = 5F; + private static final Double DOUBLE_PROPERTY = 6D; + private static final String STRING_PROPERTY = "Hello World!"; + + public static void addPropertiesToMessage(Message message) throws JMSException { + message.setBooleanProperty("booleanProperty", BOOLEAN_PROPERTY); + message.setByteProperty("byteProperty", BYTE_PROPERTY); + message.setIntProperty("intProperty", INT_PROPERTY); + message.setLongProperty("longProperty", LONG_PROPERTY); + message.setFloatProperty("floatProperty", FLOAT_PROPERTY); + message.setDoubleProperty("doubleProperty", DOUBLE_PROPERTY); + message.setStringProperty("stringProperty", STRING_PROPERTY); + } + + public static void addPropertiesToRecord(StructuredRecord.Builder builder, Schema schema) { + Schema.Type propertiesFieldType = schema.getField(JMSMessageParts.PROPERTIES).getSchema().getType(); + if (propertiesFieldType.equals(Schema.Type.STRING)) { + withStringPropertiesField(builder); + } else if (propertiesFieldType.equals(Schema.Type.RECORD)) { + withRecordPropertiesField(builder, schema); + } + } + + private static void withStringPropertiesField(StructuredRecord.Builder builder) { + HashMap properties = new HashMap<>(); + properties.put("booleanProperty", BOOLEAN_PROPERTY); + properties.put("byteProperty", BYTE_PROPERTY); + properties.put("intProperty", INT_PROPERTY); + properties.put("longProperty", LONG_PROPERTY); + properties.put("floatProperty", FLOAT_PROPERTY); + properties.put("doubleProperty", DOUBLE_PROPERTY); + properties.put("stringProperty", STRING_PROPERTY); + builder.set(JMSMessageParts.PROPERTIES, new Gson().toJson(properties)); + } + + private static void withRecordPropertiesField(StructuredRecord.Builder builder, Schema schema) { + StructuredRecord propertiesRecord = StructuredRecord + .builder(schema.getField(JMSMessageParts.PROPERTIES).getSchema()) + .set("booleanProperty", BOOLEAN_PROPERTY) + .set("byteProperty", BYTE_PROPERTY) + .set("intProperty", INT_PROPERTY) + .set("longProperty", LONG_PROPERTY) + .set("floatProperty", FLOAT_PROPERTY) + .set("doubleProperty", DOUBLE_PROPERTY) + .set("stringProperty", STRING_PROPERTY) + .build(); + builder.set(JMSMessageParts.PROPERTIES, propertiesRecord); + } + + public static Schema.Field getPropertiesField() { + return Schema.Field.of(JMSMessageParts.PROPERTIES, Schema.recordOf( + JMSMessageParts.PROPERTIES, + Schema.Field.of("booleanProperty", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("byteProperty", Schema.of(Schema.Type.BYTES)), + Schema.Field.of("intProperty", Schema.of(Schema.Type.INT)), + Schema.Field.of("longProperty", Schema.of(Schema.Type.LONG)), + Schema.Field.of("floatProperty", Schema.of(Schema.Type.FLOAT)), + Schema.Field.of("doubleProperty", Schema.of(Schema.Type.DOUBLE)), + Schema.Field.of("stringProperty", Schema.of(Schema.Type.STRING)) + ) + ); + } +} diff --git a/suppressions.xml b/suppressions.xml new file mode 100644 index 0000000..f1ba54a --- /dev/null +++ b/suppressions.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + diff --git a/widgets/JMS-batchsink.json b/widgets/JMS-batchsink.json new file mode 100644 index 0000000..694de63 --- /dev/null +++ b/widgets/JMS-batchsink.json @@ -0,0 +1,101 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "display-name": "JMS", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + }, + { + "widget-type": "textbox", + "label": "Connection Factory", + "name": "connectionFactory", + "widget-attributes": { + "default": "ConnectionFactory" + } + }, + { + "widget-type": "textbox", + "label": "JMS Username", + "name": "jmsUsername" + }, + { + "widget-type": "password", + "label": "JMS Password", + "name": "jmsPassword" + }, + { + "widget-type": "text", + "label": "Provider URL", + "name": "providerUrl", + "widget-attributes": { + "placeholder": "tcp://hostname:61616" + } + }, + { + "widget-type": "select", + "label": "Type", + "name": "type", + "widget-attributes": { + "values": [ + "Queue", + "Topic" + ], + "default": "Queue" + } + }, + { + "widget-type": "textbox", + "label": "Destination Queue or Topic Name", + "name": "destinationName" + }, + { + "widget-type": "textbox", + "label": "JNDI Context Factory", + "name": "jndiContextFactory", + "widget-attributes": { + "default": "org.apache.activemq.jndi.ActiveMQInitialContextFactory" + } + }, + { + "widget-type": "textbox", + "label": "JNDI Username", + "name": "jndiUsername" + }, + { + "widget-type": "password", + "label": "JNDI Password", + "name": "jndiPassword" + }, + { + "widget-type": "select", + "label": "Message Type", + "name": "messageType", + "widget-attributes": { + "values": [ + "Message", + "Text", + "Bytes", + "Map", + "Object" + ], + "default": "Text" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": {} + } + ] +} diff --git a/widgets/JMS-streamingsource.json b/widgets/JMS-streamingsource.json new file mode 100644 index 0000000..f1350b2 --- /dev/null +++ b/widgets/JMS-streamingsource.json @@ -0,0 +1,207 @@ +{ + "metadata": { + "spec-version": "1.5" + }, + "display-name": "JMS", + "configuration-groups": [ + { + "label": "Basic", + "properties": [ + { + "widget-type": "textbox", + "label": "Reference Name", + "name": "referenceName" + }, + { + "widget-type": "textbox", + "label": "Connection Factory", + "name": "connectionFactory", + "widget-attributes": { + "default": "ConnectionFactory" + } + }, + { + "widget-type": "textbox", + "label": "JMS Username", + "name": "jmsUsername" + }, + { + "widget-type": "password", + "label": "JMS Password", + "name": "jmsPassword" + }, + { + "widget-type": "text", + "label": "Provider URL", + "name": "providerUrl", + "widget-attributes": { + "placeholder": "tcp://hostname:61616" + } + }, + { + "widget-type": "select", + "label": "Type", + "name": "type", + "widget-attributes": { + "values": [ + "Queue", + "Topic" + ], + "default": "Queue" + } + }, + { + "widget-type": "textbox", + "label": "Source Queue or Topic Name", + "name": "sourceName" + }, + { + "widget-type": "textbox", + "label": "JNDI Context Factory", + "name": "jndiContextFactory", + "widget-attributes": { + "default": "org.apache.activemq.jndi.ActiveMQInitialContextFactory" + } + }, + { + "widget-type": "textbox", + "label": "JNDI Username", + "name": "jndiUsername" + }, + { + "widget-type": "password", + "label": "JNDI Password", + "name": "jndiPassword" + }, + { + "widget-type": "toggle", + "name": "messageHeader", + "label": "Keep Message Header", + "widget-attributes": { + "default": "true", + "on": { + "value": "true", + "label": "True" + }, + "off": { + "value": "false", + "label": "False" + } + } + }, + { + "widget-type": "toggle", + "name": "messageProperties", + "label": "Keep Message Properties", + "widget-attributes": { + "default": "true", + "on": { + "value": "true", + "label": "True" + }, + "off": { + "value": "false", + "label": "False" + } + } + }, + { + "widget-type": "select", + "label": "Message Type", + "name": "messageType", + "widget-attributes": { + "values": [ + "Message", + "Text", + "Bytes", + "Map", + "Object" + ], + "default": "Text" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "default-schema": { + "name": "etlSchemaBody", + "type": "record", + "fields": [ + { + "name": "header", + "type": { + "type": "record", + "name": "header", + "fields": [ + { + "name": "messageId", + "type": ["string", "null"], + "default": null + }, + { + "name": "messageTimestamp", + "type": ["long", "null"], + "default": null + }, + { + "name": "correlationId", + "type": ["string", "null"], + "default": null + }, + { + "name": "replyTo", + "type": ["string", "null"], + "default": null + }, + { + "name": "destination", + "type": ["string", "null"], + "default": null + }, + { + "name": "deliveryNode", + "type": ["int", "null"], + "default": null + }, + { + "name": "redelivered", + "type": ["boolean", "null"], + "default": null + }, + { + "name": "type", + "type": ["boolean", "null"], + "default": null + }, + { + "name": "expiration", + "type": ["long", "null"], + "default": null + }, + { + "name": "priority", + "type": ["int", "null"], + "default": null + } + ] + } + }, + { + "name": "body", + "type": "string" + }, + { + "name": "properties", + "type": "string" + } + ] + } + } + } + ] +}