From af2ea1521c690f5974252008f57db3b24c2e210d Mon Sep 17 00:00:00 2001 From: Andrew Brown Date: Mon, 4 Mar 2024 19:08:45 +0000 Subject: [PATCH] kinesis data streams --- kinesis/datastreams/Readme.md | 104 ++++++++++++++++ kinesis/datastreams/deploy | 13 ++ kinesis/datastreams/kpl-example/pom.xml | 114 ++++++++++++++++++ .../src/main/java/co/exampro/App.java | 68 +++++++++++ .../src/test/java/co/exampro/AppTest.java | 20 +++ .../target/classes/co/exampro/App.class | Bin 0 -> 1848 bytes .../test-classes/co/exampro/AppTest.class | Bin 0 -> 467 bytes kinesis/datastreams/template.yaml | 11 ++ .../classes/com/example/myapp/App.class | Bin 0 -> 927 bytes .../com/example/myapp/DependencyFactory.class | Bin 0 -> 1836 bytes .../com/example/myapp/AppTest.class | Bin 0 -> 834 bytes 11 files changed, 330 insertions(+) create mode 100644 kinesis/datastreams/Readme.md create mode 100755 kinesis/datastreams/deploy create mode 100644 kinesis/datastreams/kpl-example/pom.xml create mode 100644 kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java create mode 100644 kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java create mode 100644 kinesis/datastreams/kpl-example/target/classes/co/exampro/App.class create mode 100644 kinesis/datastreams/kpl-example/target/test-classes/co/exampro/AppTest.class create mode 100755 kinesis/datastreams/template.yaml create mode 100644 s3/sdk/java/myapp/target/classes/com/example/myapp/App.class create mode 100644 s3/sdk/java/myapp/target/classes/com/example/myapp/DependencyFactory.class create mode 100644 s3/sdk/java/myapp/target/test-classes/com/example/myapp/AppTest.class diff --git a/kinesis/datastreams/Readme.md b/kinesis/datastreams/Readme.md new file mode 100644 index 0000000..b268fc6 --- /dev/null +++ b/kinesis/datastreams/Readme.md @@ -0,0 +1,104 @@ +## KPL + +Generate a new java project. +This will create a new directory call kpl-example with a bare bones java project. + +```sh +mvn archetype:generate \ +-DgroupId=co.exampro \ +-DartifactId=kpl-example \ +-DarchetypeArtifactId=maven-archetype-quickstart \ +-DarchetypeVersion=1.4 \ +-DinteractiveMode=false +``` + +In the pom.xml we will add into the project tag profiles tag +and change our build directory for the outputed binary. + +You will have to change the directory based on your developer enviroment. +In our configuration we are using Gitpod so we are leveraging an env var. +You could hardcode this value to go where you need it to go. + +```xml + + + dev + + ${env.THEIA_WORKSPACE_ROOT}/kinesis/datastreams/kpl-example/output + + + +``` + +We need to add the KPL to our pom.xml depenednecies + +https://central.sonatype.com/artifact/com.amazonaws/amazon-kinesis-producer + +```xml + + com.amazonaws + amazon-kinesis-producer + 0.15.9 + +``` + +To package the dependencies will need to use Maven Shade Plugin +- provides the capability to package the artifact in an uber-jar, including its dependencies and to shade + +```xml + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + + co.exampro.App + + + true + application + + + + package + + shade + + + + +``` + +Build your binary + +```sh +mvn clean package shade:shade -Pdev +``` + +Run your binary + +cd $THEIA_WORKSPACE_ROOT/output +java -jar application-1.0-SNAPSHOT-shaded.jar + +# Set your env vars + +export DATA_STREAM_NAME="datastream-MyStream-mVSDDqynVfJj" +export DATA_STREAM_PARTITION_KEY="456" +export DATA_STREAM_SHARD="shardId-000000000000" + +# Test your Datastream using the AWS CLI + +Write data +```sh +echo 'Send reinforcements' | base64 +aws kinesis put-record --stream-name $DATA_STREAM_NAME --partition-key $DATA_STREAM_PARTITION_KEY --data U2VuZCByZWluZm9yY2VtZW50cwo= +``` + +Get data + +```sh +export SHARD_ITERATOR=$(aws kinesis get-shard-iterator --shard-id $DATA_STREAM_SHARD --shard-iterator-type TRIM_HORIZON --stream-name $DATA_STREAM_NAME --query 'ShardIterator') +aws kinesis get-records --shard-iterator $SHARD_ITERATOR +``` \ No newline at end of file diff --git a/kinesis/datastreams/deploy b/kinesis/datastreams/deploy new file mode 100755 index 0000000..7265ccf --- /dev/null +++ b/kinesis/datastreams/deploy @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +echo "== deploy Data Stream" + +STACK_NAME="datastream" + +# https://awscli.amazonaws.com/v2/documentation/api/latest/reference/cloudformation/deploy/index.html +aws cloudformation deploy \ +--template-file template.yaml \ +--capabilities CAPABILITY_NAMED_IAM \ +--no-execute-changeset \ +--region ca-central-1 \ +--stack-name $STACK_NAME \ No newline at end of file diff --git a/kinesis/datastreams/kpl-example/pom.xml b/kinesis/datastreams/kpl-example/pom.xml new file mode 100644 index 0000000..6fac81d --- /dev/null +++ b/kinesis/datastreams/kpl-example/pom.xml @@ -0,0 +1,114 @@ + + + + 4.0.0 + + + + dev + + ${env.THEIA_WORKSPACE_ROOT}/kinesis/datastreams/kpl-example/output + + + + + co.exampro + kpl-example + 1.0-SNAPSHOT + + kpl-example + + http://www.example.com + + + UTF-8 + 1.7 + 1.7 + + + + + junit + junit + 4.11 + test + + + com.amazonaws + amazon-kinesis-producer + 0.15.9 + + + + + + + + + maven-clean-plugin + 3.1.0 + + + + maven-resources-plugin + 3.0.2 + + + maven-compiler-plugin + 3.8.0 + + + maven-surefire-plugin + 2.22.1 + + + maven-jar-plugin + 3.0.2 + + + maven-install-plugin + 2.5.2 + + + maven-deploy-plugin + 2.8.2 + + + + maven-site-plugin + 3.7.1 + + + maven-project-info-reports-plugin + 3.0.0 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.2 + + + + co.exampro.App + + + true + application + + + + package + + shade + + + + + + + + + diff --git a/kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java b/kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java new file mode 100644 index 0000000..fbaa4c8 --- /dev/null +++ b/kinesis/datastreams/kpl-example/src/main/java/co/exampro/App.java @@ -0,0 +1,68 @@ +package co.exampro; +import com.amazonaws.services.kinesis.producer.KinesisProducer; +import java.nio.ByteBuffer; + +import com.amazonaws.services.kinesis.producer.UserRecordResult; +import com.amazonaws.services.kinesis.producer.Attempt; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.LinkedList; +import java.util.concurrent.Future; +import java.util.concurrent.ExecutionException; // For handling the exceptions from Future.get() +import java.nio.charset.StandardCharsets; // For using StandardCharsets.UTF_8 + + +public class App +{ + public static void main( String[] args ) + { + // Need to update these! + String streamName = System.getenv("DATA_STREAM_NAME"); + String partitionKey = System.getenv("DATA_STREAM_PARTITION_KEY"); + + System.out.println("Hello World!"); + + System.out.println("DATA_STREAM_NAME: " + streamName); + System.out.println("DATA_PARTITION_KEY: " + partitionKey); + + KinesisProducer kinesis = new KinesisProducer(); + List> putFutures = new LinkedList>(); + + for (int i = 0; i < 5; ++i) { + ByteBuffer data = null; + System.out.println("LOOP"); + try { + data = ByteBuffer.wrap("myData".getBytes("UTF-8")); + } catch (java.io.UnsupportedEncodingException e) { + e.printStackTrace(); // handle the exception here + } + // doesn't block + + putFutures.add( + kinesis.addUserRecord(streamName, partitionKey, data) + ); + } + + + for (Future f : putFutures) { + try { + UserRecordResult result = f.get(); // This does block + if (result.isSuccessful()) { + System.out.println("Put record into shard " + result.getShardId()); + } else { + for (Attempt attempt : result.getAttempts()) { + // Analyze and respond to the failure + System.out.println("Record Put Failed"); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); // handle the InterruptedException + System.out.println("Interrupted Exception"); + } catch (ExecutionException e) { + System.out.println("Execution Exception"); + e.printStackTrace(); // handle the ExecutionException + } + } + System.out.println( "Goodbye World!" ); + } +} diff --git a/kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java b/kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java new file mode 100644 index 0000000..ed866bd --- /dev/null +++ b/kinesis/datastreams/kpl-example/src/test/java/co/exampro/AppTest.java @@ -0,0 +1,20 @@ +package co.exampro; + +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +/** + * Unit test for simple App. + */ +public class AppTest +{ + /** + * Rigorous Test :-) + */ + @Test + public void shouldAnswerWithTrue() + { + assertTrue( true ); + } +} diff --git a/kinesis/datastreams/kpl-example/target/classes/co/exampro/App.class b/kinesis/datastreams/kpl-example/target/classes/co/exampro/App.class new file mode 100644 index 0000000000000000000000000000000000000000..d657ad9af4fdca6e5cf3a7e000a927bf5beb3603 GIT binary patch literal 1848 zcmbVM-%}e^6#i~WvLRhs48Mw6OSPo|O1Iid+fbFHgo1?dBLPHITy`(%(qwmMv!URF zGtTry{|le`TpvQm&iDuT7QXc^a45E1C!)%SF*qT6L{aeG%9zCi4uWn2EI0v)N5HG#+@$CL?lpi@N@ zF@cMD%a(}Ptb&?UQx@8OWr#AEdW-t1l zWtS&M*5Y`JUelRUW~*3Q$z>L{3Ympm0zK$c(A$Q%cu@0YRYgAr2)itOY3~UPr_Km_ z+7!p2z~J#bOPQ6@Txo8xur;6C=w!loRJ@DJB*Lkam%e5U%h8qy)hA%7S104RBB0L7 zO2rvkcf5*uHG!-6K*9S0J>m7*9aLPybpgc&-&t_i6x_3Zx&1?cQm{sujP0ef;E%3Lo#n3o4^g_aBi9)uA8 zhfo+)Sp3fsn@M&2)2t^r;1PlvZt}wt|5tSwvnwphg6@VRO~E>ES9I#0A!n>$?!`Gp zZv_`*$8$<>)B@+YfU{U~iyNMe>Fv_Q);`sg)4zf0J~ARJuAKEM5dEBFyZ zAq%t3n_>;?j2FXmd_f;-1>fUK?#?k<7Mu8rdFR<6U*j9Hvy2WR|wu9%Oc9f(mT37(i#WJ6(9nQw}|o2s0l zo&5c|Pvv}SifkO~{6*R)<)#B$5UKM}Xs70KsB%T9SZm{iQ$`c1kr+GSvx0`><^25+ zO|%2F$l5w_o=fW@9O0ND6sv5__PfvJKH;M&Y$|V6IeE0X#btR0oPn06qX8bhXjzH@ zWt^0eRxjSM^R}`>ilq&N*rOP&8%XylV>lq!Lw(cLAT2>Xa#kS!1BY8sNUfk{e6;?B JosjF{^cRF4UD*Ht literal 0 HcmV?d00001 diff --git a/kinesis/datastreams/template.yaml b/kinesis/datastreams/template.yaml new file mode 100755 index 0000000..991945c --- /dev/null +++ b/kinesis/datastreams/template.yaml @@ -0,0 +1,11 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: A Simple Kinesis Data Streams Provisioned +Resources: + MyStream: + Type: AWS::Kinesis::Stream + Properties: + # Name: MyStream + # RetentionPeriodHours: 24 + ShardCount: 1 + StreamModeDetails: + StreamMode: PROVISIONED \ No newline at end of file diff --git a/s3/sdk/java/myapp/target/classes/com/example/myapp/App.class b/s3/sdk/java/myapp/target/classes/com/example/myapp/App.class new file mode 100644 index 0000000000000000000000000000000000000000..791c77aeb03f0d47d831346587b5e6889ae4c7ad GIT binary patch literal 927 zcmb7CO>Yx15PeS4Y!cEoq=Z8GXrY$`1U|I4R1qks;t+vEpm%38v@W~eW!Kv_{35>O*AIX)04nPP9)cqDcL}tnQI`0$ubl=B9_q2{U!XbFhep{f;bT+2^gr+U^8@W& zP^YBUXo!3KP*t`iwyt|G{^}Lp^G9KSg`-g(1MLT2*uG-rGuA%ks2=+jW&<76u*T>f zu5;98rt|B#!K=VcY!tPg@apiDhaa71JW$Wk8##09JX|Jh8qW>D7YcaJ^nUxAT^hvbxAjMcbTCn8rxxLPFh(RK@-9%TF}bS zkdBbhuo2NQP-ryoq3+F0^;rHM7M%2hRG|yRP>;iLCF+!BP(r&#jCkp zso>%f+{A!_euj+z@1o%DbIq1SzN-tv;j3m*w=FYAa*ed)srsF56?KY8#UO4`IS46S zVtF$-zJK&U%njkLf;$Xz0kPL3yqdX+dl+Wu{wuRu=nS8tK_$EpYvRbMlYKm}2dEV^ zi#kK|+l5!_$2~LtNv$wpj9^T`DD_}Ka<<&lEz{wf{;uKy#u>uf>D|>tdTV!Up;@L$mz=WZ+qRKBkzSL& z!k%|69x}WMZnnm4jn+!x_1T5{3P z@#~juQN?3CVQBIDh@poPKR**`JVnx0!PexO?#;Wt3$t{_gD|M;M6aDRgyTp@!E=V$ z%L-CAF9k0cVhzJ+f>l{ou)?r@8L*)SsPT(o&?kVtDL{z+H2O(aNaD0J+FK^Rq3w&W zj?n6Bvn8^`D?<8}x?G4vJp%ZU1q$CxT()_*MH z5hHEqX99%aIT<83pbl+9FB$E_01e+!^ka(7Jjb5#5T5$$GsJ1{B6$hRc-b08g6;|J Fe*hbbGrL z5lDOhAB7k@kq{(YW<0yk@4eZXpTEBS0Pq<19Ml-vQJQdhA(AYXJozZHjQ6taL>9(D zouN4q=Yq#VpYm73iHrT^o$$W~x(o+np+~Xo%QH)k-WB82#-qM8HrL0gE{xD-AYw}z z?Y`AUC32t&MIt?|QzMKb5W{Ym=BIpO$u`AI8JTmDDPDHjbr3LYEi~a_11m1Js7V=( z)3R*&xP~^v+F!;P-W-PizErpBOeEq{s>O$b7cxIrku2ynli^74d}^3eBSv1B?$JDh zhaKE-u*=X{+(EA()~ZV%H*t%>7uCtLGb>{1{9uu-Iiuwm-S<{UB@;ty@knSKja(Ks zrX2|8y)t;OQ=3QfxhiMw&$j4MsfOF2b)l^TYV<{GLeZg$Q)ZNx?_a|GTwQw<%WDu| zm7+fdP+p^o;R@Z>us%aID;yw*z7%ME$Hv=BTn!%to9qj=Uj&;EzvB8I>Hu|ut->Y3 hCQ)t^VJXv@{yvj&NuX1?Zjy~V*sFZ+6Rw84zX3OM&W8X1 literal 0 HcmV?d00001