feat: spooler uats

.github/workflows/uat.yaml

@@ -0,0 +1,32 @@
+name: OTF UATS
+
+on:
+  pull_request:
+    branches: '*'
+
+env:
+  AWS_REGION : "us-west-2"
+  CODE_BUILD_PROJECT_LINUX: "DiskSpoolerUatCodeBuildLinux"
+  AWS_ROLE_TO_ASSUME: "arn:aws:iam::686385081908:role/aws-greengrass-disk-spooler-codebuild-uat-role-amazonlinux"
+
+jobs:
+  uat-linux:
+    permissions:
+      id-token: write
+      contents: read
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ ubuntu-latest ]
+    steps:
+      - name: configure aws credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.AWS_ROLE_TO_ASSUME }}
+          role-session-name: nucleusCI
+          aws-region: ${{ env.AWS_REGION }}
+      - name: Run UAT on linux
+        uses: aws-actions/aws-codebuild-run-build@v1
+        with:
+          project-name: ${{ env.CODE_BUILD_PROJECT_LINUX }}
+          buildspec-override: uat/codebuild/uat_linux_buildspec.yaml

.gitignore

@@ -0,0 +1,6 @@
+# UATs
+uat-results
+greengrass-nucleus-latest.zip
+
+# GDK
+greengrass-build

pom.xml

@@ -94,6 +94,7 @@
                             <excludes>
                                 <exclude>*</exclude>
                                 <exclude>codestyle/**</exclude>
+                                <exclude>uat-results/**</exclude>
                                 <exclude>src/*/resources/**</exclude>
                             </excludes>
                         </licenseSet>

run-uats.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+set -e
+curl -s https://d2s8p88vqu9w66.cloudfront.net/releases/greengrass-nucleus-latest.zip > greengrass-nucleus-latest.zip
+mvn -U -ntp clean package -DskipTests
+mvn -U -ntp clean verify -f uat/pom.xml
+sudo java -Dggc.archive=greengrass-nucleus-latest.zip -Dtest.log.path=uat-results -Dtags=DiskSpooler -jar uat/testing-features/target/greengrass-disk-spooler-testing-features.jar

uat/README.md

@@ -0,0 +1,49 @@
+## Disk Spooler User Acceptance Tests
+
+User Acceptance Tests for Disk Spooler run using `aws-greengrass-testing-standalone` as a library. They execute E2E
+tests which will spin up an instance of Greengrass on your device and execute different sets of tests, by installing
+the `aws.greengrass.DiskSpooler` component.
+
+## Running UATs locally
+
+Ensure credentials are available by setting them in environment variables. In unix based systems:
+
+```bash
+export AWS_ACCESS_KEY_ID=<AWS_ACCESS_KEY_ID>
+export AWS_SECRET_ACCESS_KEY=<AWS_SECRET_ACCESS_KEY>
+```
+
+on Windows Powershell
+
+```bash
+$Env:AWS_ACCESS_KEY_ID=<AWS_ACCESS_KEY_ID>
+$Env:AWS_SECRET_ACCESS_KEY=<AWS_SECRET_ACCESS_KEY>
+```
+
+For UATs to run you will need to package your entire application along with `aws-greengrass-testing-standalone` into
+an uber jar. To do run (from the root of the project)
+
+```
+mvn -U -ntp clean verify -f uat/pom.xml
+```
+
+Note: Everytime you make changes to the codebase you will have to rebuild the uber jar for those changes to be present
+on the final artifact.
+
+Finally, download the zip containing the latest version of the Nucleus, which will be used to provision Greengrass for
+the UATs.
+
+```bash
+curl -s https://d2s8p88vqu9w66.cloudfront.net/releases/greengrass-nucleus-latest.zip > greengrass-nucleus-latest.zip
+```
+
+Execute the UATs by running the following command from the root of the project.
+
+```
+sudo java -Dggc.archive=<path-to-nucleus-zip> -Dtest.log.path=<path-to-test-results-folder> -Dtags=DiskSpooler -jar uat/testing-features/target/greengrass-disk-spooler-testing-features.jar
+```
+
+Command arguments:
+
+Dggc.archive - path to the nucleus zip that was downloaded
+Dtest.log.path - path where you would like the test results to be stored 

uat/codebuild/uat_linux_buildspec.yaml

@@ -0,0 +1,29 @@
+#
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+
+version: 0.2
+phases:
+  install:
+    runtime-versions:
+      java: corretto11
+  build:
+    commands:
+      - curl -s https://d2s8p88vqu9w66.cloudfront.net/releases/greengrass-nucleus-latest.zip > /tmp/greengrass-nucleus-latest.zip
+      - mvn -U -ntp verify -DskipTests=true
+      - mvn -U -ntp clean verify -f uat/pom.xml
+      - java -Dggc.archive=/tmp/greengrass-nucleus-latest.zip
+        -Dtags='DiskSpooler' -Dggc.install.root=$CODEBUILD_SRC_DIR -Dggc.log.level=INFO -Daws.region=$AWS_REGION
+        -jar uat/testing-features/target/greengrass-disk-spooler-testing-features.jar
+
+artifacts:
+  files:
+    - 'testResults/**/*'
+  name: 'DiskSpoolerUatLinuxLogs.zip'
+
+reports:
+  uat-reports:
+    files:
+      - "TEST-greengrass-results.xml"
+    file-format: "JUNITXML"

uat/custom-components/pom.xml

@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+    Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+    SPDX-License-Identifier: Apache-2.0
+
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <artifactId>custom-components</artifactId>
+    <groupId>com.aws.greengrass</groupId>
+    <version>1.0-SNAPSHOT</version>
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.4</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>custom-components</finalName>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>com.aws.greengrass.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+    <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.12.7.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>1.7.30</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk.iotdevicesdk</groupId>
+            <artifactId>aws-iot-device-sdk</artifactId>
+            <version>1.17.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+            <version>1.7.30</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.13</version>
+            <scope>compile</scope>
+        </dependency>
+    </dependencies>
+</project>

uat/custom-components/src/main/java/com/aws/greengrass/Main.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.aws.greengrass;
+
+import java.util.function.Consumer;
+
+public final class Main {
+    private Main(){}
+
+    @SuppressWarnings({"unchecked", "deprecation"})
+    public static void main(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException {
+        ((Consumer<String[]>) Class.forName("com.aws.greengrass.artifacts." + System.getProperty("componentName"))
+                .newInstance()).accept(args);
+    }
+}

uat/custom-components/src/main/java/com/aws/greengrass/artifacts/IotMqttPublisher.java

@@ -0,0 +1,73 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.aws.greengrass.artifacts;
+
+import com.aws.greengrass.utils.Client;
+import com.aws.greengrass.utils.IPCTestUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import software.amazon.awssdk.aws.greengrass.GreengrassCoreIPC;
+import software.amazon.awssdk.aws.greengrass.GreengrassCoreIPCClientV2;
+import software.amazon.awssdk.aws.greengrass.model.PublishToIoTCoreRequest;
+import software.amazon.awssdk.aws.greengrass.model.QOS;
+import software.amazon.awssdk.aws.greengrass.model.ReportedLifecycleState;
+import software.amazon.awssdk.aws.greengrass.model.UnauthorizedError;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.function.Consumer;
+
+public class IotMqttPublisher implements Consumer<String[]> {
+    private static final Logger LOGGER = LoggerFactory.getLogger(IotMqttPublisher.class);
+    private static final String SPOOL_SIZE_ERROR = "Message is larger than the size of message spool.";
+    private static final String SPOOL_FULL_ERROR = "Message spool is full. Message could not be added.";
+
+    @Override
+    public void accept(String[] args) {
+        try (Client assertionClient = new Client();
+             GreengrassCoreIPCClientV2 eventStreamRpcConnection = IPCTestUtils.getGreengrassClient()) {
+            GreengrassCoreIPC greengrassCoreIPCClient = eventStreamRpcConnection.getClient();
+
+            String topic = System.getenv("topic");
+            String payload = System.getenv("payload");
+            QOS qos = IPCTestUtils.getQOSFromValue(Integer.parseInt(System.getenv("qos")));
+
+            PublishToIoTCoreRequest request = new PublishToIoTCoreRequest();
+            request.setTopicName(topic);
+            request.setPayload(payload.getBytes(StandardCharsets.UTF_8));
+            request.setQos(qos);
+
+            try {
+                greengrassCoreIPCClient.publishToIoTCore(request, Optional.empty()).getResponse().get();
+                assertionClient.sendAssertion(true, "Successfully published to IoT topic " + topic, "");
+            } catch (ExecutionException e) {
+                LOGGER.error("Error occurred while publishing to IoT topic", e);
+                try {
+                    String errorMessage = e.getCause().getMessage();
+                    if (errorMessage.contains(SPOOL_SIZE_ERROR) || errorMessage.contains(SPOOL_FULL_ERROR)) {
+                        assertionClient.sendAssertion(true, "SPOOL_FULL_ERROR", "Spooler is full");
+                        return;
+                    }
+                    if (e.getCause() instanceof UnauthorizedError) {
+                        assertionClient.sendAssertion(true, e.getCause().getMessage(), "Unauthorized error while publishing to IoT topic " + topic);
+                        return;
+                    }
+                } catch (IOException e2) {
+                    LOGGER.error("Failed to send assertion", e2);
+                }
+                IPCTestUtils.reportState(greengrassCoreIPCClient, ReportedLifecycleState.ERRORED);
+            }
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            System.exit(1);
+        } catch (Exception e) {
+            LOGGER.error("Service errored", e);
+            System.exit(1);
+        }
+    }
+}

uat/custom-components/src/main/java/com/aws/greengrass/utils/Client.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.aws.greengrass.utils;
+
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.entity.ContentType;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.impl.client.HttpClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class Client implements AutoCloseable {
+    private static final Logger LOGGER = LoggerFactory.getLogger(Client.class);
+
+    private final CloseableHttpClient httpClient;
+
+    public Client() {
+        httpClient = HttpClients.custom()
+                .setRetryHandler(new DefaultHttpRequestRetryHandler(5, false)).build();
+    }
+
+    @Override
+    public void close() {
+        if (httpClient != null) {
+            try {
+                httpClient.close();
+            } catch (IOException e) {
+                 LOGGER.warn("unable to close http client", e);
+            }
+        }
+    }
+
+    public void sendAssertion(boolean success, String context, String message) throws IOException {
+        int defaultPort = (int) Double.parseDouble(System.getProperty("serverPort"));
+        sendAssertionWithCustomizedPort(success, context, message, defaultPort);
+    }
+
+    public void sendAssertionWithCustomizedPort(boolean success, String context, String message, int port)
+            throws IOException {
+        HttpPost httpPost = new HttpPost(
+                "http://localhost:" + port + "/assert");
+        httpPost.setEntity(new ByteArrayEntity(
+                (String.format("{\"success\": %s, \"context\": \"%s\", \"message\": \"%s\"}", success, context,
+                        message)).getBytes(StandardCharsets.UTF_8), ContentType.APPLICATION_JSON));
+        try (CloseableHttpResponse response = httpClient.execute(httpPost)) {
+            response.getStatusLine().getStatusCode();
+        }
+    }
+}