Terminate Zombie Build Instances #1072
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Required since we cannot guarantee instances are always terminated. | |
| # Also a failsafe against a dev writing a workflow that does not terminate build instances. | |
| # Though the average Airbyte build runtime as of this commit is ~20 mins, connector builds | |
| # can take up to 3 hours. Set this to 3 hours to include these longer runs. | |
| name: Terminate Zombie Build Instances | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 */1 * * *" | |
| jobs: | |
| terminate: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: List and Terminate Instances Older Than 4 Hours | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| # See https://github.com/aws/aws-cli/issues/5623 | |
| AWS_EC2_METADATA_DISABLED: true | |
| run: | | |
| set -euxo pipefail | |
| export TIME_LIMIT=14400 # 4 hours | |
| aws configure set default.region us-east-2 | |
| # See https://awscli.amazonaws.com/v2/documentation/api/latest/reference/ec2/describe-instances.html for describe command. | |
| # Since the AWS cli returns an ISO HH:MM timestamp, and Jq only accepts Z timestamps, we define a function toZ to convert this. | |
| export to_terminate=$(aws ec2 describe-instances --no-paginate --filters --filters 'Name=tag:Repository,Values=airbytehq/airbyte-platform-internal,airbytehq/airbyte-platform,airbytehq/airbyte' Name=instance-state-name,Values=running \ | |
| --query 'Reservations[*].Instances[*].{Instance:InstanceId,LaunchTime:LaunchTime}' --output json \ | |
| | jq 'def toZ(str): str | (split("+")[0] + "Z") | fromdate ; | |
| flatten | map( { InstanceId: .Instance, LaunchTime: toZ(.LaunchTime) } ) | map( select ( .LaunchTime < (now - (env.TIME_LIMIT|tonumber)) ) )') | |
| echo "MARKED FOR TERMINATION: ${to_terminate}" | |
| # See https://docs.aws.amazon.com/cli/latest/reference/ec2/terminate-instances.html for terminate command. | |
| echo $to_terminate | jq '.[] | .InstanceId' | xargs --no-run-if-empty --max-args=1 aws ec2 terminate-instances --instance-ids | |
| terminate-github-instances: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v3 | |
| - name: List and Terminate GH actions in status 'offline' | |
| env: | |
| GITHUB_PAT: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} | |
| run: ./tools/bin/gh_action_zombie_killer |