Adding ALB troubleshooting scenario and Troubleshooting Methodologies

manifests/modules/troubleshooting/alb/.workshop/terraform/main.tf

@@ -0,0 +1,176 @@
+terraform {
+  required_providers {
+    #    kubectl = {
+    #      source  = "gavinbunney/kubectl"
+    #      version = ">= 1.14"
+    #    }
+  }
+}
+
+
+
+provider "aws" {
+  region = "us-east-1"
+  alias  = "virginia"
+}
+
+locals {
+  tags = {
+    module = "troubleshooting"
+  }
+}
+
+data "aws_vpc" "selected" {
+  tags = {
+    created-by = "eks-workshop-v2"
+    env        = var.addon_context.eks_cluster_id
+  }
+}
+
+data "aws_subnets" "public" {
+  tags = {
+    created-by = "eks-workshop-v2"
+    env        = var.addon_context.eks_cluster_id
+  }
+
+  filter {
+    name   = "tag:Name"
+    values = ["*Public*"]
+  }
+}
+
+
+resource "time_sleep" "blueprints_addons_sleep" {
+  depends_on = [
+    module.eks_blueprints_addons
+  ]
+
+  create_duration  = "15s"
+  destroy_duration = "15s"
+}
+
+
+resource "null_resource" "break_public_subnet" {
+  triggers = {
+    #cluster_id     = var.addon_context.eks_cluster_id
+    public_subnets = join(" ", data.aws_subnets.public.ids)
+    timestamp      = timestamp()
+  }
+  count = length(data.aws_subnets.public)
+
+  provisioner "local-exec" {
+    when    = destroy
+    command = "aws ec2 create-tags --resources ${self.triggers.public_subnets} --tags Key=kubernetes.io/role/elb,Value='1'"
+  }
+
+  provisioner "local-exec" {
+    command = "aws ec2 delete-tags --resources ${self.triggers.public_subnets} --tags Key=kubernetes.io/role/elb,Value='1'"
+  }
+}
+
+
+module "eks_blueprints_addons" {
+  source  = "aws-ia/eks-blueprints-addons/aws"
+  version = "1.16.2"
+
+  enable_aws_load_balancer_controller = true
+  aws_load_balancer_controller = {
+    wait = true
+  }
+
+  cluster_name      = var.addon_context.eks_cluster_id
+  cluster_endpoint  = var.addon_context.aws_eks_cluster_endpoint
+  cluster_version   = var.eks_cluster_version
+  oidc_provider_arn = var.addon_context.eks_oidc_provider_arn
+
+  tags = merge(
+    var.tags,
+    local.tags
+  )
+
+  depends_on = [null_resource.break_public_subnet]
+
+}
+
+
+# create a new policy from json file 
+resource "aws_iam_policy" "issue" {
+  name   = "issue"
+  path   = "/"
+  policy = file("${path.module}/template/other_issue.json")
+}
+
+# attach issue policy to role
+resource "aws_iam_role_policy_attachment" "issue_policy_attachment" {
+  role       = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_name
+  policy_arn = aws_iam_policy.issue.arn
+  depends_on = [module.eks_blueprints_addons, time_sleep.blueprints_addons_sleep]
+}
+
+resource "null_resource" "detach_existing_policy" {
+  triggers = {
+    role_name = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_name,
+    timestamp = timestamp()
+  }
+
+  provisioner "local-exec" {
+    command = "aws iam detach-role-policy --role-name ${self.triggers.role_name} --policy-arn ${module.eks_blueprints_addons.aws_load_balancer_controller.iam_policy_arn}"
+    when    = create
+  }
+
+  depends_on = [aws_iam_role_policy_attachment.issue_policy_attachment]
+}
+
+resource "null_resource" "kustomize_app" {
+  triggers = {
+    always_run = timestamp()
+  }
+
+  provisioner "local-exec" {
+    command = "kubectl apply -k ~/environment/eks-workshop/modules/troubleshooting/alb/creating-alb"
+    when    = create
+  }
+
+  depends_on = [aws_iam_role_policy_attachment.issue_policy_attachment]
+}
+
+
+
+# Example to now how to get variables from add ons outputs DO-NOT-DELETE; AddOns and helms documentaitons does not show exactly the output variables returned
+#resource "null_resource" "blue_print_output" {
+#  for_each = module.eks_blueprints_addons.aws_load_balancer_controller
+#  triggers = {
+#
+#    timestamp      = timestamp()
+#  }
+#
+#  #count = length(module.eks_blueprints_addons.aws_load_balancer_controller)
+#  provisioner "local-exec" {
+#    command = "mkdir -p /eks-workshop/logs; echo \" key: ${each.key} Value:${each.value}\" >> /eks-workshop/logs/action-load-balancer-output.log"
+#  }
+#
+#  depends_on = [module.eks_blueprints_addons,time_sleep.blueprints_addons_sleep]
+#}
+
+#option to run a bash script file
+#resource "null_resource" "break2" {
+#  provisioner "local-exec" {
+#    command = "${path.module}/template/break.sh ${path.module} mod2"    
+#  }
+#
+#  triggers = {
+#    always_run = timestamp()
+#  }
+#  depends_on = [module.eks_blueprints_addons,time_sleep.blueprints_addons_sleep]
+#}
+
+#option to run a kubectl manifest
+#resource "kubectl_manifest" "alb" {
+#  yaml_body = templatefile("${path.module}/template/ingress.yaml", {
+#
+#  })
+#
+#  depends_on = [null_resource.break_policy]
+#}
+
+

manifests/modules/troubleshooting/alb/.workshop/terraform/outputs.tf

@@ -0,0 +1,13 @@
+output "environment_variables" {
+  description = "Environment variables to be added to the IDE shell"
+  value = merge({
+    VPC_ID                                    = data.aws_vpc.selected.id,
+    LOAD_BALANCER_CONTROLLER_ROLE_NAME        = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_name,
+    LOAD_BALANCER_CONTROLLER_POLICY_ARN_FIX   = module.eks_blueprints_addons.aws_load_balancer_controller.iam_policy_arn,
+    LOAD_BALANCER_CONTROLLER_POLICY_ARN_ISSUE = aws_iam_policy.issue.arn,
+    LOAD_BALANCER_CONTROLLER_ROLE_ARN         = module.eks_blueprints_addons.aws_load_balancer_controller.iam_role_arn
+    }, {
+    for index, id in data.aws_subnets.public.ids : "PUBLIC_SUBNET_${index + 1}" => id
+    }
+  )
+}

manifests/modules/troubleshooting/alb/.workshop/terraform/template/break.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+#. .env
+
+set -e
+
+mkdir -p /eks-workshop/logs
+log_file=/eks-workshop/logs/action-$(date +%s).log
+
+exec 2>&1
+
+logmessage() {
+  echo "$@" >&7
+  echo "$@" >&1
+}
+export -f logmessage
+
+# Function to get the role name from a role ARN
+get_role_name_from_arn() {
+    local role_arn=$1
+
+    # Extract the role name from the ARN
+    role_name=$(logmessage "$role_arn" | awk -F'/' '{print $NF}')
+
+    if [ -n "$role_name" ]; then
+        logmessage "$role_name"
+    else
+        logmessage "Failed to retrieve role name from ARN: $role_arn"
+        return 1
+    fi
+}
+
+# Function to get the Kubernetes role attached to a service account
+get_service_account_role() {
+    local namespace=$1
+    local service_account=$2
+
+    # Get the role ARN associated with the service account
+    role_arn=$(kubectl get serviceaccount "$service_account" -n "$namespace" -o jsonpath="{.metadata.annotations['eks\.amazonaws\.com\/role-arn']}")
+
+    if [ -n "$role_arn" ]; then
+        logmessage "Service Account: $service_account"
+        logmessage "Namespace: $namespace"
+        logmessage "Role ARN: $role_arn"
+        get_role_name_from_arn "$role_arn"
+        return 0
+    else
+        logmessage "Failed to retrieve role for service account '$service_account' in namespace '$namespace'"
+        return 1
+    fi
+
+}
+
+# Function to get the first policy ARN attached to a role ARN
+get_first_policy_arn_from_role_arn() {
+    local role_arn=$1
+
+    # Get the list of policies attached to the role
+    policy_arn=$(aws iam list-attached-role-policies --role-name "$role_arn" --query 'AttachedPolicies[0].PolicyArn' --output text)
+
+    if [ -n "$policy_arn" ]; then
+        logmessage "First Policy ARN attached to role '$role_arn':"
+        logmessage "Policy: $policy_arn"
+        return 0
+    else
+        logmessage "Failed to retrieve policy ARN for role '$role_arn'"
+        return 1
+    fi    
+}
+
+# Function to update the policy with new statement
+update_policy_with_new_statement() {
+    local policy_arn=$1
+    local new_statement=$2
+
+    logmessage "PolicyARN: $policy_arn"
+    logmessage "Statement: $new_statement"
+    aws iam create-policy-version --policy-arn $policy_arn --policy-document $new_statement --set-as-default
+
+}
+
+# Function to remove an action from a policy statement
+remove_action_from_policy_statement() {
+    local policy_name=$1
+    local action_to_remove=$2
+
+    # Get the current policy document
+    policy_document=$(aws iam get-policy-version --policy-arn "$policy_arn" --query 'PolicyVersion.Document' --version-id v1 --output json)
+
+    # Remove the specified action from the statements
+    new_statements=$(logmessage "$policy_document" | jq ".Statement[] | select(.Action[] | contains('$action_to_remove')) | .Action = [.Action[] | select(. != '$action_to_remove')]")
+    new_policy_document=$(logmessage '{"Version": "2012-10-17", "Statement": '"$new_statements"'}')
++
+    # Update the policy with the modified document
+    logmessage "Policy Document"
+    logmessage $new_policy_document
+    #aws iam create-policy-version --policy-arn "$policy_arn" --policy-document "$new_policy_document" --set-as-default
+
+    if [ $? -eq 0 ]; then
+        logmessage "Action removed from policy statement successfully."
+        return 0
+    else
+        logmessage "Failed to remove action from policy statement."
+        return 1
+    fi
+}
+
+# Function to remove tags from subnets ids
+remove_tags_from_subnets() {
+    local tag_key="Key=kubernetes.io/role/elb,Value=1"
+
+    logmessage "retrive subnets ids with tag key assigned to specific vpc_id via aws cli"    
+    logmessage "getting public subnets from VPC: $vpc_id "
+
+
+    subnets_vpc=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" --query 'Subnets[*].SubnetId' --output text)
+    logmessage "subnets_vpc: $subnets_vpc"
+
+
+#remove tag from subnets with AWS cli
+    for subnet_id in $subnets_vpc; do
+        logmessage "public subnets: $subnet_id"
+        aws ec2 delete-tags --resources "$subnet_id" --tags "Key=$tag_key" || logmessage "Failed to remove tag from subnet $subnet_id"
+    done
+    return 0
+}
+
+# Getting the service role
+path_tofile=$1
+mode=$2
+vpc_id=$3
+public_subnets=$4
+namespace="kube-system"
+service_account="aws-load-balancer-controller-sa"
+#new_statement="file://$path_tofile/template/iam_policy_incorrect.json"
+new_statement="file://$path_tofile/template/other_issue.json"
+
+logmessage "path_sent: $path_tofile"
+
+
+# validate if mode is equal to mod1
+logmessage "mode: $mode"
+if [ "$mode" == "mod1" ]; then
+    logmessage "Removing subnet tags"
+    remove_tags_from_subnets
+else
+    logmessage "Removing permissions"
+    get_service_account_role "$namespace" "$service_account"
+    get_first_policy_arn_from_role_arn "$role_name"
+    update_policy_with_new_statement "$policy_arn" "$new_statement"
+
+fi
+
+
+
+