VPC endpoints added;s3 state for e2e tests

.github/workflows/delete-log-groups.py

@@ -0,0 +1,22 @@
+import os
+import boto3
+
+REGION = os.environ.get('AWS_DEFAULT_REGION', 'us-west-2')
+CLIENT = boto3.client('logs', region_name=REGION)
+
+
+def delete_log_groups():
+    """Delete all log groups in the region that start with `/aws/eks/`"""
+    response = CLIENT.describe_log_groups(
+        logGroupNamePrefix='/aws/eks/',
+        limit=50
+    )
+
+    for log_group in [log.get('logGroupName') for log in response.get('logGroups', {})]:
+        CLIENT.delete_log_group(
+            logGroupName=log_group
+        )
+
+
+if __name__ == '__main__':
+    delete_log_groups()

.github/workflows/e2e-parallel-destroy.yml

@@ -0,0 +1,56 @@
+name: e2e-parallel-destroy-only
+
+on:
+  workflow_dispatch:
+    inputs:
+      TFDestroy:
+        description: 'Destroy TF Automatically (false/true) - Default: true'
+        required: true
+        default: 'true'
+
+concurrency: e2e-parallel-destroy
+
+jobs:
+  deploy:
+    name: Run e2e test
+    runs-on: ubuntu-latest
+
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - example_path: analytics/terraform/emr-eks-karpenter
+          - example_path: analytics/terraform/spark-k8s-operator
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup backend
+        # Un-comment remote backend for use in workflow
+        run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf
+
+      - name: Auth AWS
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+          role-duration-seconds: 3600
+          role-session-name: GithubActions-Session
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.0.0
+
+      - name: Terraform Destroy
+        working-directory: ${{ matrix.example_path }}
+        run: |
+          terraform init -upgrade=true
+          terraform destroy -target=module.eks_blueprints_kubernetes_addons  -no-color -input=false -auto-approve
+          terraform destroy -target=module.eks_blueprints  -no-color -input=false -auto-approve
+          terraform destroy -no-color -input=false -auto-approve

.github/workflows/e2e-parallel-full.yml

@@ -0,0 +1,158 @@
+name: e2e-parallel-full
+
+on:
+  workflow_dispatch:
+    inputs:
+      TFDestroy:
+        description: 'Destroy TF Automatically (false/true) - Default: true'
+        required: true
+        default: 'true'
+
+concurrency: e2e-parallel-full
+
+env:
+  IAMLIVE_VERSION: v0.48.0
+  BUCKET_NAME: doeks-iam-policies-examples
+
+jobs:
+  prereq-cleanup:
+    name: Prerequisite Cleanup
+    runs-on: ubuntu-latest
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Auth AWS
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+          role-duration-seconds: 3600
+          role-session-name: GithubActions-Session
+
+      - name: Ensure log groups are removed
+        run: |
+          pip3 install boto3
+          python3 .github/workflows/delete-log-groups.py
+
+  deploy:
+    name: Run e2e test
+    runs-on: ubuntu-latest
+    needs: prereq-cleanup
+
+    # These permissions are needed to interact with GitHub's OIDC Token endpoint.
+    permissions:
+      id-token: write
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - example_path: analytics/terraform/emr-eks-karpenter
+          - example_path: analytics/terraform/spark-k8s-operator
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup backend
+        # Un-comment remote backend for use in workflow
+        run: sed -i "s/# //g" ${{ matrix.example_path }}/versions.tf
+
+      - name: Auth AWS
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+          role-duration-seconds: 3600
+          role-session-name: GithubActions-Session
+
+      - name: Iamlive Setup & Run
+        run: |
+          #!/bin/bash
+          set -eox pipefail
+          wget -O iamlive.tar.gz "https://github.com/iann0036/iamlive/releases/download/${{ env.IAMLIVE_VERSION }}/iamlive-${{ env.IAMLIVE_VERSION }}-linux-amd64.tar.gz"
+          tar -xzf iamlive.tar.gz
+          chmod +x iamlive
+          IAMLIVE_PID=$(./iamlive --mode csm --output-file ${HOME}/policy.json --refresh-rate 1 --sort-alphabetical --force-wildcard-resource --background)
+          echo "iamlive_pid=$IAMLIVE_PID" >> $GITHUB_ENV
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+        with:
+          terraform_version: 1.0.0
+
+      - name: Terraform Apply
+        id: apply
+        working-directory: ${{ matrix.example_path }}
+        run: |
+          terraform init -upgrade=true
+          export AWS_CSM_ENABLED=true
+          export AWS_CSM_PORT=31000
+          export AWS_CSM_HOST=127.0.0.1
+          terraform apply -target=module.vpc -no-color -input=false -auto-approve
+          terraform apply -target=module.vpc_endpoints_sg -no-color -input=false -auto-approve
+          terraform apply -target=module.vpc_endpoints -no-color -input=false -auto-approve
+          terraform apply -target=module.eks_blueprints -no-color -input=false -auto-approve
+          terraform apply -target=module.eks_blueprints_kubernetes_addons -no-color -input=false -auto-approve
+          terraform apply -no-color -input=false -auto-approve
+
+      - name: Terraform Destroy
+        if: github.event.inputs.TFDestroy == 'true' && (steps.apply.outcome == 'success' || steps.apply.outcome == 'failure')
+        working-directory: ${{ matrix.example_path }}
+        run: |
+          terraform init -upgrade=true
+          export AWS_CSM_ENABLED=true
+          export AWS_CSM_PORT=31000
+          export AWS_CSM_HOST=127.0.0.1
+          terraform destroy -target=module.eks_blueprints_kubernetes_addons  -no-color -input=false -auto-approve
+          terraform destroy -target=module.eks_blueprints  -no-color -input=false -auto-approve
+          terraform destroy -no-color -input=false -auto-approve
+
+      - name: Fail if TF apply failed
+        if: steps.apply.outcome == 'failure'
+        run: |
+          echo "Terraform Apply step failed...Please check the logs of the Terraform Apply step."
+          echo "Failing the job to avoid false positives."
+          kill ${{ env.iamlive_pid }}
+          while $(kill -0 ${{ env.iamlive_pid }} 2>/dev/null); do sleep 1; done;
+          cat ${HOME}/policy.json
+          exit 1
+
+      - name: Iamlive Print & Upload Policy
+        if: ${{ always() }}
+        run: |
+          kill ${{ env.iamlive_pid }}
+          while $(kill -0 ${{ env.iamlive_pid }} 2>/dev/null); do sleep 1; done;
+          cat ${HOME}/policy.json
+          aws s3 cp ${HOME}/policy.json s3://${{ env.BUCKET_NAME }}/${{ matrix.example_path }}.json
+
+  post_deploy:
+    if: ${{ always() }}
+    needs: [deploy]
+    permissions:
+      id-token: write
+      contents: read
+    name: Merge Policies and Print Final IAM Policy
+    runs-on: ubuntu-latest
+    steps:
+      # Be careful not to change this to explicit checkout from PR ref/code, as below we run a python code that may change from the PR code.
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Configure AWS credentials from Test account
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          role-to-assume: ${{ secrets.ROLE_TO_ASSUME }}
+          aws-region: us-west-2
+          role-duration-seconds: 3600
+          role-session-name: GithubActions-Session
+
+      - name: Merge iamlive IAM policies and Print Final Policy
+        id: dirs
+        run: |
+          pip3 install boto3
+          python3 .github/workflows/iam-policy-generator.py

.github/workflows/iam-policy-generator.py

@@ -0,0 +1,35 @@
+import json
+import boto3
+import os
+
+iam_actions = []
+s3 = boto3.resource('s3')
+bucket_name = os.getenv('BUCKET_NAME')
+bucket = s3.Bucket(bucket_name)
+bucket_files = [x.key for x in bucket.objects.all()]
+
+# Read all the files from the bucket
+for file in bucket_files:
+    obj = s3.Object(bucket_name, file)
+    f = obj.get()['Body'].read()
+    data = json.loads(f)
+    # Merge all policies actions, keep them unique with 'set'
+    for statement in data['Statement']:
+        iam_actions = list(set(iam_actions + statement['Action']))
+
+# Skeleton IAM policy template , wild card all resources for now.
+template = {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+            ],
+            "Resource": "*"
+        }
+    ]
+}
+
+# Apply merged actions to the skeleton IAM policy
+template['Statement'][0]['Action'] = sorted(iam_actions)
+print(json.dumps(template, indent=4))

analytics/cdk/emr-eks/bin/emr-eks.ts

@@ -13,7 +13,7 @@ const region = process.env.CDK_DEFAULT_REGION;
 
 const executionRolePolicyStatement: PolicyStatement[] = [
   new PolicyStatement({
-    actions:['logs:PutLogEvents','logs:CreateLogStream','logs:DescribeLogGroups','logs:DescribeLogStreams'], 
+    actions:['logs:PutLogEvents','logs:CreateLogStream','logs:DescribeLogGroups','logs:DescribeLogStreams'],
     resources:['arn:aws:logs:*:*:*'],
   }),
 ];

analytics/cdk/emr-eks/lib/emr-eks-blueprint-stack.ts

@@ -1,12 +1,12 @@
 import { Construct } from 'constructs';
 import * as blueprints from '@aws-quickstart/eks-blueprints'
-import { 
-  DirectVpcProvider, 
-  GenericClusterProvider, 
-  GlobalResources, 
-  PlatformTeam, 
-  EmrEksTeamProps, 
-  EmrEksTeam, 
+import {
+  DirectVpcProvider,
+  GenericClusterProvider,
+  GlobalResources,
+  PlatformTeam,
+  EmrEksTeamProps,
+  EmrEksTeam,
   EmrEksAddOn
 } from '@aws-quickstart/eks-blueprints';
 import { CapacityType, ClusterLoggingTypes, KubernetesVersion, NodegroupAmiType } from 'aws-cdk-lib/aws-eks';

analytics/terraform/emr-eks-ack/README.md → analytics/terraform/emr-eks-ack-crossplane/README.md

@@ -1,5 +1,6 @@
-# EMR on EKS with ACK Controller
-Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/amazon-emr-on-eks/emr-eks-ack) to deploy this pattern and run sample tests.
+# EMR on EKS with ACK Controller and Crossplane
+This pattern is used to deploy the EKS Cluster with EMR on EKS ACK Controllers and Crossplane.
+Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/amazon-emr-on-eks/emr-eks-ack-crossplane) to deploy this pattern and run sample tests.
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
@@ -24,6 +25,8 @@ Checkout the [documentation website](https://awslabs.github.io/data-on-eks/docs/
 | <a name="module_eks_blueprints"></a> [eks\_blueprints](#module\_eks\_blueprints) | github.com/aws-ia/terraform-aws-eks-blueprints | v4.15.0 |
 | <a name="module_eks_blueprints_kubernetes_addons"></a> [eks\_blueprints\_kubernetes\_addons](#module\_eks\_blueprints\_kubernetes\_addons) | github.com/aws-ia/terraform-aws-eks-blueprints//modules/kubernetes-addons | v4.15.0 |
 | <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 3.0 |
+| <a name="module_vpc_endpoints"></a> [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | ~> 3.0 |
+| <a name="module_vpc_endpoints_sg"></a> [vpc\_endpoints\_sg](#module\_vpc\_endpoints\_sg) | terraform-aws-modules/security-group/aws | ~> 4.0 |
 
 ## Resources
 

analytics/terraform/emr-eks-ack-crossplane/locals.tf

@@ -0,0 +1,13 @@
+locals {
+  name   = var.name
+  region = var.region
+
+  vpc_cidr      = var.vpc_cidr
+  azs           = slice(data.aws_availability_zones.available.names, 0, 3)
+  vpc_endpoints = ["autoscaling", "ecr.api", "ecr.dkr", "ec2", "ec2messages", "elasticloadbalancing", "sts", "kms", "logs", "ssm", "ssmmessages"]
+
+  tags = merge(var.tags, {
+    Blueprint  = local.name
+    GithubRepo = "github.com/awslabs/data-on-eks"
+  })
+}

analytics/terraform/emr-eks-ack/main.tf → analytics/terraform/emr-eks-ack-crossplane/main.tf

@@ -1,16 +1,3 @@
-locals {
-  name   = var.name
-  region = var.region
-
-  vpc_cidr = var.vpc_cidr
-  azs      = slice(data.aws_availability_zones.available.names, 0, 3)
-
-  tags = merge(var.tags, {
-    Blueprint  = local.name
-    GithubRepo = "github.com/awslabs/data-on-eks"
-  })
-}
-
 #---------------------------------------------------------------
 # EKS Blueprints
 #---------------------------------------------------------------

analytics/terraform/emr-eks-ack/versions.tf → analytics/terraform/emr-eks-ack-crossplane/versions.tf

@@ -15,4 +15,11 @@ terraform {
       version = ">= 2.4.1"
     }
   }
+
+  # ##  Used for end-to-end testing on project; update to suit your needs
+  # backend "s3" {
+  #   bucket = "doeks-github-actions-e2e-test-state"
+  #   region = "us-west-2"
+  #   key    = "e2e/emr-eks-ack-crossplane/terraform.tfstate"
+  # }
 }