Merge pull request #4190 from nulib/test-localstack-community-edition

Use LocalStack Community Edition for tests

.github/workflows/test.yml

@@ -148,22 +148,22 @@ jobs:
         ports:
           - 9200:9200
       localstack:
-        image: localstack/localstack-pro:2.1.0
+        image: localstack/localstack
         env:
           DOCKER_HOST: unix:///var/run/docker.sock
           GATEWAY_LISTEN: 0.0.0.0:4566
-          LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }}
         ports:
           - 4566:4566
         volumes:
           - /var/run/docker.sock:/var/run/docker.sock:ro
     steps:
       - uses: actions/checkout@v2
-      - name: Provision Localstack using Cloud Pod
+      - uses: hashicorp/setup-terraform@v3
+      - name: Provision Localstack using Terraform
         run: |
-          pip install localstack==2.1.0
-          curl -O https://nul-public.s3.amazonaws.com/meadow/test/localstack.pod
-          localstack pod load file://$PWD/localstack.pod
+          terraform init
+          terraform apply -auto-approve -var-file test.tfvars -var localstack_endpoint=https://localhost.localstack.cloud:4566
+        working-directory: ./infrastructure/localstack
       - uses: actions/setup-node@v3
         with:
           node-version-file: ./.tool-versions

infrastructure/localstack/digester.tf

@@ -0,0 +1,11 @@
+module "digester_function" {
+  source = "./modules/localstack_lambda"
+
+  description   = "Function to tag an S3 object with its md5 checksum"
+  function_name = "digest-tag"
+  handler       = "index.handler"
+  memory_size   = 1024
+  runtime       = "nodejs14.x"
+  source_dir    = "${path.module}/lambdas/digest-tag"
+  timeout       = 120
+}

infrastructure/localstack/environment_config.tf

@@ -0,0 +1,51 @@
+locals {
+  project       = "meadow"
+  port_offset   = 0 # terraform.workspace == "test" ? 2 : 1
+
+  computed_secrets = {
+    db   = {
+      host        = "localhost"
+      port        = 5432 + local.port_offset
+      user        = "docker"
+      password    = "d0ck3r"
+    }
+
+    index = {
+      index_endpoint    = "http://localhost:${9200 + local.port_offset}"
+      kibana_endpoint   = "http://localhost:${5601 + local.port_offset}"
+    }
+    ldap = {
+      host       = "localhost"
+      base       = "DC=library,DC=northwestern,DC=edu"
+      port       = 389 + local.port_offset
+      user_dn    = "cn=Administrator,cn=Users,dc=library,dc=northwestern,dc=edu"
+      password   = "d0ck3rAdm1n!"
+      ssl        = "false"
+    }
+  }
+
+  config_secrets = merge(var.config_secrets, local.computed_secrets)
+}
+
+resource "aws_secretsmanager_secret" "config_secrets" {
+  name    = "config/meadow"
+  description = "Meadow configuration secrets"
+}
+
+resource "aws_secretsmanager_secret" "ssl_certificate" {
+  name = "config/wildcard_ssl"
+  description = "Wildcard SSL certificate and private key"
+}
+
+resource "aws_secretsmanager_secret_version" "config_secrets" {
+  secret_id = aws_secretsmanager_secret.config_secrets.id
+  secret_string = jsonencode(local.config_secrets)
+}
+
+resource "aws_secretsmanager_secret_version" "ssl_certificate" {
+  secret_id       = aws_secretsmanager_secret.ssl_certificate.id
+  secret_string   = jsonencode({
+    certificate = file(var.ssl_certificate_file)
+    key         = file(var.ssl_key_file)
+  })
+}

infrastructure/localstack/lambdas/digest-tag/index.js

@@ -0,0 +1,56 @@
+const AWS = require("aws-sdk");
+const crypto = require("crypto");
+
+AWS.config.update({
+  httpOptions: {
+    timeout: 600000
+  }
+});
+
+const handler = async (event, _context, _callback) => {
+  const record = event.Records[0].s3;
+  const bucket = record.bucket.name;
+  const key = record.object.key;
+  const digests = await generateDigest(bucket, key);
+  console.log("Digest result:", digests);
+  const s3 = new AWS.S3();
+  const tags = {
+    TagSet: [{Key: "computed-md5", Value: digests.md5}, {Key: "computed-md5-last-modified", Value: Number(new Date()).toString()}]
+  };
+  const tagResult = await s3.putObjectTagging({
+    Bucket: bucket,
+    Key: key,
+    Tagging: tags
+  }).promise();
+  console.log(tagResult);
+  return tags;
+};
+
+const generateDigest = (bucket, key) => {
+  console.log(`Digesting s3://${bucket}/${key}`);
+  return new Promise((resolve, reject) => {
+    let md5 = crypto.createHash("md5");
+
+    let s3Stream = new AWS.S3()
+      .getObject({
+        Bucket: bucket,
+        Key: key
+      })
+      .createReadStream();
+
+    s3Stream
+      .on("data", (chunk) => {
+        md5.update(chunk);
+      })
+      .on("end", () =>
+        resolve({
+          md5: md5.digest("hex")
+        })
+      )
+      .on("error", (err) => reject(err));
+  });
+};
+
+module.exports = {
+  handler
+};