From 578193be74df9ebe0843007e3cb8f73d6f487d87 Mon Sep 17 00:00:00 2001 From: Rahul Vats Date: Fri, 29 Dec 2023 13:30:45 +0530 Subject: [PATCH 1/5] adding task to delete regression cluster --- .../amazon/aws/example_dags/example_aws_nuke.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py index e41185baa..d679f7fd2 100644 --- a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py +++ b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py @@ -22,6 +22,12 @@ SLACK_USERNAME = os.getenv("SLACK_USERNAME", "airflow_app") SLACK_WEBHOOK_CONN = os.getenv("SLACK_WEBHOOK_CONN", "http_slack") +REGRESSION_CLUSTER_AWS_ACCESS_KEY = os.getenv("REGRESSION_CLUSTER_AWS_ACCESS_KEY", "**********") +REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY = os.getenv("REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY", "***********") +REGRESSION_CLUSTER_AWS_DEFAULT_REGION = os.getenv("REGRESSION_CLUSTER_AWS_DEFAULT_REGION", "us-east-1") + + + def generate_task_report(**context: Any) -> None: """Generate a report of the task statuses for the DAG run and send it to configured Slack channel for alerts.""" @@ -125,6 +131,15 @@ def check_dag_status(**kwargs: Any) -> None: f"aws emr-containers list-virtual-clusters --state RUNNING --region {AWS_DEFAULT_REGION} | jq -r '.virtualClusters[].id' | xargs -I % aws emr-containers delete-virtual-cluster --id % --region {AWS_DEFAULT_REGION}; ", ) + terminate_regression_clusters = BashOperator( + task_id="terminate_regression_clusters", + bash_command=f"set -e; " + f"aws configure set aws_access_key_id {REGRESSION_CLUSTER_AWS_ACCESS_KEY}; " + f"aws configure set aws_secret_access_key {REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY}; " + f"aws configure set default.region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}; " + f"for cluster in $(aws eks list-clusters --region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION} --output json | jq -r '.clusters[]'); do aws eks delete-cluster --name '$cluster' --region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}; done" + ) + execute_aws_nuke = BashOperator( task_id="execute_aws_nuke", bash_command=f"aws configure set aws_access_key_id {AWS_ACCESS_KEY_ID}; " @@ -163,6 +178,7 @@ def check_dag_status(**kwargs: Any) -> None: start >> [get_airflow_version, get_airflow_executor] >> terminate_running_emr_virtual_clusters + >> terminate_regression_clusters >> execute_aws_nuke >> delete_stale_emr_vpcs >> delete_stale_emr_iam_roles From 8be57423a16902cf756b0b3ff06c067496a42287 Mon Sep 17 00:00:00 2001 From: Rahul Vats Date: Fri, 29 Dec 2023 16:59:06 +0530 Subject: [PATCH 2/5] updating nuke dag to delete eks cluster from dag authoring account --- .../delete_eks_cluster_and_nodes.sh | 39 +++++++++++++++++++ .../aws/example_dags/example_aws_nuke.py | 14 +++---- 2 files changed, 46 insertions(+), 7 deletions(-) create mode 100755 astronomer/providers/amazon/aws/example_dags/delete_eks_cluster_and_nodes.sh diff --git a/astronomer/providers/amazon/aws/example_dags/delete_eks_cluster_and_nodes.sh b/astronomer/providers/amazon/aws/example_dags/delete_eks_cluster_and_nodes.sh new file mode 100755 index 000000000..1e58da0f5 --- /dev/null +++ b/astronomer/providers/amazon/aws/example_dags/delete_eks_cluster_and_nodes.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Check if the region parameter is provided +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +AWS_REGION="$1" + +# List all EKS clusters +clusters=$(aws eks list-clusters --region $AWS_REGION | jq -r '.clusters[]') + +# Loop through each EKS cluster +for cluster in $clusters; do + echo "Processing EKS cluster: $cluster" + + # List nodegroups for the cluster + nodegroups=$(aws eks list-nodegroups --cluster-name $cluster --region $AWS_REGION | jq -r '.nodegroups[]') + + # Delete each nodegroup + for nodegroup in $nodegroups; do + echo "Deleting nodegroup '$nodegroup' for cluster '$cluster'" + aws eks delete-nodegroup --cluster-name $cluster --nodegroup-name $nodegroup --region $AWS_REGION + aws eks wait nodegroup-deleted --cluster-name $cluster --nodegroup-name $nodegroup --region $AWS_REGION + done + + + # Delete the EKS cluster + echo "Deleting EKS cluster: $cluster" + aws eks delete-cluster --name $cluster --region $AWS_REGION + + # Wait for the EKS cluster to be deleted + echo "Waiting for EKS cluster '$cluster' to be deleted..." + aws eks wait cluster-deleted --name $cluster --region $AWS_REGION + +done + +echo "Script execution completed." diff --git a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py index d679f7fd2..8bd928bcf 100644 --- a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py +++ b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py @@ -23,12 +23,12 @@ SLACK_WEBHOOK_CONN = os.getenv("SLACK_WEBHOOK_CONN", "http_slack") REGRESSION_CLUSTER_AWS_ACCESS_KEY = os.getenv("REGRESSION_CLUSTER_AWS_ACCESS_KEY", "**********") -REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY = os.getenv("REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY", "***********") +REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY = os.getenv( + "REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY", "***********" +) REGRESSION_CLUSTER_AWS_DEFAULT_REGION = os.getenv("REGRESSION_CLUSTER_AWS_DEFAULT_REGION", "us-east-1") - - def generate_task_report(**context: Any) -> None: """Generate a report of the task statuses for the DAG run and send it to configured Slack channel for alerts.""" dag_run = context["dag_run"] @@ -134,10 +134,10 @@ def check_dag_status(**kwargs: Any) -> None: terminate_regression_clusters = BashOperator( task_id="terminate_regression_clusters", bash_command=f"set -e; " - f"aws configure set aws_access_key_id {REGRESSION_CLUSTER_AWS_ACCESS_KEY}; " - f"aws configure set aws_secret_access_key {REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY}; " - f"aws configure set default.region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}; " - f"for cluster in $(aws eks list-clusters --region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION} --output json | jq -r '.clusters[]'); do aws eks delete-cluster --name '$cluster' --region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}; done" + f"aws configure set aws_access_key_id {REGRESSION_CLUSTER_AWS_ACCESS_KEY}; " + f"aws configure set aws_secret_access_key {REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY}; " + f"aws configure set default.region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}; " + f"sh $AIRFLOW_HOME/dags/delete_eks_cluster_and_nodes.sh {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}", ) execute_aws_nuke = BashOperator( From 04e3113984f58a4fd0e64ebd3073c91d3ed0e063 Mon Sep 17 00:00:00 2001 From: Rahul Vats Date: Fri, 29 Dec 2023 17:19:12 +0530 Subject: [PATCH 3/5] updating task name --- .../providers/amazon/aws/example_dags/example_aws_nuke.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py index 8bd928bcf..7e7d8b255 100644 --- a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py +++ b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py @@ -131,8 +131,8 @@ def check_dag_status(**kwargs: Any) -> None: f"aws emr-containers list-virtual-clusters --state RUNNING --region {AWS_DEFAULT_REGION} | jq -r '.virtualClusters[].id' | xargs -I % aws emr-containers delete-virtual-cluster --id % --region {AWS_DEFAULT_REGION}; ", ) - terminate_regression_clusters = BashOperator( - task_id="terminate_regression_clusters", + terminate_dag_authoring_regression_clusters = BashOperator( + task_id="terminate_dag_authoring_regression_clusters", bash_command=f"set -e; " f"aws configure set aws_access_key_id {REGRESSION_CLUSTER_AWS_ACCESS_KEY}; " f"aws configure set aws_secret_access_key {REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY}; " @@ -178,7 +178,7 @@ def check_dag_status(**kwargs: Any) -> None: start >> [get_airflow_version, get_airflow_executor] >> terminate_running_emr_virtual_clusters - >> terminate_regression_clusters + >> terminate_dag_authoring_regression_clusters >> execute_aws_nuke >> delete_stale_emr_vpcs >> delete_stale_emr_iam_roles From a5bebb28ec499931f065831b909c6cfe83aaac21 Mon Sep 17 00:00:00 2001 From: Rahul Vats Date: Fri, 29 Dec 2023 18:13:39 +0530 Subject: [PATCH 4/5] renaming eks deleteion file name --- ...uster_and_nodes.sh => example_delete_eks_cluster_and_nodes.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename astronomer/providers/amazon/aws/example_dags/{delete_eks_cluster_and_nodes.sh => example_delete_eks_cluster_and_nodes.sh} (100%) diff --git a/astronomer/providers/amazon/aws/example_dags/delete_eks_cluster_and_nodes.sh b/astronomer/providers/amazon/aws/example_dags/example_delete_eks_cluster_and_nodes.sh similarity index 100% rename from astronomer/providers/amazon/aws/example_dags/delete_eks_cluster_and_nodes.sh rename to astronomer/providers/amazon/aws/example_dags/example_delete_eks_cluster_and_nodes.sh From ec75c803896c47b7a34354c8b762a93c8aaa59c1 Mon Sep 17 00:00:00 2001 From: Rahul Vats Date: Fri, 29 Dec 2023 18:14:57 +0530 Subject: [PATCH 5/5] update file name in aws nuke dag --- .../providers/amazon/aws/example_dags/example_aws_nuke.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py index 7e7d8b255..eb6950f54 100644 --- a/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py +++ b/astronomer/providers/amazon/aws/example_dags/example_aws_nuke.py @@ -137,7 +137,7 @@ def check_dag_status(**kwargs: Any) -> None: f"aws configure set aws_access_key_id {REGRESSION_CLUSTER_AWS_ACCESS_KEY}; " f"aws configure set aws_secret_access_key {REGRESSION_CLUSTER_AWS_SECRET_ACCESS_KEY}; " f"aws configure set default.region {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}; " - f"sh $AIRFLOW_HOME/dags/delete_eks_cluster_and_nodes.sh {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}", + f"sh $AIRFLOW_HOME/dags/example_delete_eks_cluster_and_nodes.sh {REGRESSION_CLUSTER_AWS_DEFAULT_REGION}", ) execute_aws_nuke = BashOperator(