diff --git a/kinesis/datastreams/Readme.md b/kinesis/datastreams/Readme.md new file mode 100644 index 0000000..b268fc6 --- /dev/null +++ b/kinesis/datastreams/Readme.md @@ -0,0 +1,104 @@ +## KPL + +Generate a new java project. +This will create a new directory call kpl-example with a bare bones java project. + +```sh +mvn archetype:generate \ +-DgroupId=co.exampro \ +-DartifactId=kpl-example \ +-DarchetypeArtifactId=maven-archetype-quickstart \ +-DarchetypeVersion=1.4 \ +-DinteractiveMode=false +``` + +In the pom.xml we will add into the project tag profiles tag +and change our build directory for the outputed binary. + +You will have to change the directory based on your developer enviroment. +In our configuration we are using Gitpod so we are leveraging an env var. +You could hardcode this value to go where you need it to go. + +```xml + + + dev + + ${env.THEIA_WORKSPACE_ROOT}/kinesis/datastreams/kpl-example/output + + + +``` + +We need to add the KPL to our pom.xml depenednecies + +https://central.sonatype.com/artifact/com.amazonaws/amazon-kinesis-producer + +```xml + + com.amazonaws + amazon-kinesis-producer + 0.15.9 + +``` + +To package the dependencies will need to use Maven Shade Plugin +- provides the capability to package the artifact in an uber-jar, including its dependencies and to shade + +```xml + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + + co.exampro.App + + + true + application + + + + package + + shade + + + + +``` + +Build your binary + +```sh +mvn clean package shade:shade -Pdev +``` + +Run your binary + +cd $THEIA_WORKSPACE_ROOT/output +java -jar application-1.0-SNAPSHOT-shaded.jar + +# Set your env vars + +export DATA_STREAM_NAME="datastream-MyStream-mVSDDqynVfJj" +export DATA_STREAM_PARTITION_KEY="456" +export DATA_STREAM_SHARD="shardId-000000000000" + +# Test your Datastream using the AWS CLI + +Write data +```sh +echo 'Send reinforcements' | base64 +aws kinesis put-record --stream-name $DATA_STREAM_NAME --partition-key $DATA_STREAM_PARTITION_KEY --data U2VuZCByZWluZm9yY2VtZW50cwo= +``` + +Get data + +```sh +export SHARD_ITERATOR=$(aws kinesis get-shard-iterator --shard-id $DATA_STREAM_SHARD --shard-iterator-type TRIM_HORIZON --stream-name $DATA_STREAM_NAME --query 'ShardIterator') +aws kinesis get-records --shard-iterator $SHARD_ITERATOR +``` \ No newline at end of file diff --git a/kinesis/datastreams/deploy b/kinesis/datastreams/deploy new file mode 100755 index 0000000..7265ccf --- /dev/null +++ b/kinesis/datastreams/deploy @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +echo "== deploy Data Stream" + +STACK_NAME="datastream" + +# https://awscli.amazonaws.com/v2/documentation/api/latest/reference/cloudformation/deploy/index.html +aws cloudformation deploy \ +--template-file template.yaml \ +--capabilities CAPABILITY_NAMED_IAM \ +--no-execute-changeset \ +--region ca-central-1 \ +--stack-name $STACK_NAME \ No newline at end of file diff --git a/kinesis/datastreams/kpl-example/pom.xml b/kinesis/datastreams/kpl-example/pom.xml new file mode 100644 index 0000000..6fac81d --- /dev/null +++ b/kinesis/datastreams/kpl-example/pom.xml @@ -0,0 +1,114 @@ + + + + 4.0.0 + + + + dev + + ${env.THEIA_WORKSPACE_ROOT}/kinesis/datastreams/kpl-example/output + + + + + co.exampro + kpl-example + 1.0-SNAPSHOT + + kpl-example + + http://www.example.com + + + UTF-8 + 1.7 + 1.7 + + + + + junit + junit + 4.11 + test + + + com.amazonaws + amazon-kinesis-producer + 0.15.9 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + + co.exampro.App + + + true + application + + + + package + + shade + + + + + + + + + diff --git a/kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java b/kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java new file mode 100644 index 0000000..fbaa4c8 --- /dev/null +++ b/kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java @@ -0,0 +1,68 @@ +package co.exampro; +import com.amazonaws.services.kinesis.producer.KinesisProducer; +import java.nio.ByteBuffer; + +import com.amazonaws.services.kinesis.producer.UserRecordResult; +import com.amazonaws.services.kinesis.producer.Attempt; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.LinkedList; +import java.util.concurrent.Future; +import java.util.concurrent.ExecutionException; // For handling the exceptions from Future.get() +import java.nio.charset.StandardCharsets; // For using StandardCharsets.UTF_8 + + +public class App +{ + public static void main( String[] args ) + { + // Need to update these! + String streamName = System.getenv("DATA_STREAM_NAME"); + String partitionKey = System.getenv("DATA_STREAM_PARTITION_KEY"); + + System.out.println("Hello World!"); + + System.out.println("DATA_STREAM_NAME: " + streamName); + System.out.println("DATA_PARTITION_KEY: " + partitionKey); + + KinesisProducer kinesis = new KinesisProducer(); + List> putFutures = new LinkedList>(); + + for (int i = 0; i < 5; ++i) { + ByteBuffer data = null; + System.out.println("LOOP"); + try { + data = ByteBuffer.wrap("myData".getBytes("UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + e.printStackTrace(); // handle the exception here + } + // doesn't block + + putFutures.add( + kinesis.addUserRecord(streamName, partitionKey, data) + ); + } + + + for (Future f : putFutures) { + try { + UserRecordResult result = f.get(); // This does block + if (result.isSuccessful()) { + System.out.println("Put record into shard " + result.getShardId()); + } else { + for (Attempt attempt : result.getAttempts()) { + // Analyze and respond to the failure + System.out.println("Record Put Failed"); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // handle the InterruptedException + System.out.println("Interrupted Exception"); + } catch (ExecutionException e) { + System.out.println("Execution Exception"); + e.printStackTrace(); // handle the ExecutionException + } + } + System.out.println( "Goodbye World!" ); + } +} diff --git a/kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java b/kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java new file mode 100644 index 0000000..ed866bd --- /dev/null +++ b/kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java @@ -0,0 +1,20 @@ +package co.exampro; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +} diff --git a/kinesis/datastreams/kpl-example/target/classes/co/exampro/App.class b/kinesis/datastreams/kpl-example/target/classes/co/exampro/App.class new file mode 100644 index 0000000..d657ad9 Binary files /dev/null and b/kinesis/datastreams/kpl-example/target/classes/co/exampro/App.class differ diff --git a/kinesis/datastreams/kpl-example/target/test-classes/co/exampro/AppTest.class b/kinesis/datastreams/kpl-example/target/test-classes/co/exampro/AppTest.class new file mode 100644 index 0000000..adb5bfb Binary files /dev/null and b/kinesis/datastreams/kpl-example/target/test-classes/co/exampro/AppTest.class differ diff --git a/kinesis/datastreams/template.yaml b/kinesis/datastreams/template.yaml new file mode 100755 index 0000000..991945c --- /dev/null +++ b/kinesis/datastreams/template.yaml @@ -0,0 +1,11 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: A Simple Kinesis Data Streams Provisioned +Resources: + MyStream: + Type: AWS::Kinesis::Stream + Properties: + # Name: MyStream + # RetentionPeriodHours: 24 + ShardCount: 1 + StreamModeDetails: + StreamMode: PROVISIONED \ No newline at end of file diff --git a/s3/sdk/java/myapp/target/classes/com/example/myapp/App.class b/s3/sdk/java/myapp/target/classes/com/example/myapp/App.class new file mode 100644 index 0000000..791c77a Binary files /dev/null and b/s3/sdk/java/myapp/target/classes/com/example/myapp/App.class differ diff --git a/s3/sdk/java/myapp/target/classes/com/example/myapp/DependencyFactory.class b/s3/sdk/java/myapp/target/classes/com/example/myapp/DependencyFactory.class new file mode 100644 index 0000000..ca0605f Binary files /dev/null and b/s3/sdk/java/myapp/target/classes/com/example/myapp/DependencyFactory.class differ diff --git a/s3/sdk/java/myapp/target/test-classes/com/example/myapp/AppTest.class b/s3/sdk/java/myapp/target/test-classes/com/example/myapp/AppTest.class new file mode 100644 index 0000000..6873712 Binary files /dev/null and b/s3/sdk/java/myapp/target/test-classes/com/example/myapp/AppTest.class differ