Merge pull request #50 from ant-xuexiao/feat/aws-deployment

aws deployments
petercat-ai · Mar 19, 2024 · f2b2e78 · f2b2e78
2 parents 2c9d2c8 + 950b3bc
commit f2b2e78
Show file tree

Hide file tree

Showing 6 changed files with 354 additions and 5 deletions.
diff --git a/.aws/task_definition.json b/.aws/task_definition.json
@@ -0,0 +1,100 @@
+{
+    "taskDefinitionArn": "arn:aws:ecs:ap-northeast-1:654654285942:task-definition/xuexiao:160",
+    "containerDefinitions": [
+      {
+        "name": "xuexiao",
+        "image": "654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao:latest",
+        "cpu": "2048",
+        "memory": "8192",
+        "portMappings": [
+          {
+            "name": "xuexiao-5050-tcp",
+            "containerPort": 5050,
+            "hostPort": 5050,
+            "protocol": "tcp",
+            "appProtocol": "http"
+          }
+        ],
+        "essential": true,
+        "command": [
+          "uvicorn",
+          "main:app",
+          "--host",
+          "0.0.0.0",
+          "--port",
+          "5050",
+          "--workers",
+          "6"
+        ],
+        "environment": [],
+        "environmentFiles": [
+          {
+            "value": "arn:aws:s3:::xuexiao-env-variables/production.env",
+            "type": "s3"
+          }
+        ],
+        "mountPoints": [],
+        "volumesFrom": [],
+        "logConfiguration": {
+          "logDriver": "awslogs",
+          "options": {
+            "awslogs-create-group": "true",
+            "awslogs-group": "/ecs/xuexiao",
+            "awslogs-region": "ap-northeast-1",
+            "awslogs-stream-prefix": "ecs"
+          }
+        }
+      }
+    ],
+    "family": "xuexiao",
+    "taskRoleArn": "arn:aws:iam::654654285942:role/ecsTaskExecutionRole",
+    "executionRoleArn": "arn:aws:iam::654654285942:role/ecsTaskExecutionRole",
+    "networkMode": "awsvpc",
+    "revision": 160,
+    "volumes": [],
+    "status": "ACTIVE",
+    "requiresAttributes": [
+      {
+        "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
+      },
+      {
+        "name": "ecs.capability.execution-role-awslogs"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.ecr-auth"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
+      },
+      {
+        "name": "ecs.capability.env-files.s3"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.task-iam-role"
+      },
+      {
+        "name": "ecs.capability.execution-role-ecr-pull"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
+      },
+      {
+        "name": "ecs.capability.task-eni"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
+      }
+    ],
+    "placementConstraints": [],
+    "compatibilities": ["EC2", "FARGATE"],
+    "requiresCompatibilities": ["FARGATE"],
+    "cpu": "2048",
+    "memory": "8192",
+    "runtimePlatform": {
+      "cpuArchitecture": "X86_64",
+      "operatingSystemFamily": "LINUX"
+    },
+    "registeredAt": "2023-07-27T13:27:07.192Z",
+    "registeredBy": "arn:aws:iam::654654285942:root",
+    "tags": []
+  }
diff --git a/.aws/task_definition_preview.json b/.aws/task_definition_preview.json
@@ -0,0 +1,101 @@
+{
+    "taskDefinitionArn": "arn:aws:ecs:ap-northeast-1:654654285942:task-definition/botmeta-preview:4",
+    "containerDefinitions": [
+      {
+        "name": "xuexiao",
+        "image": "654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao:latest",
+        "cpu": "2048",
+        "memory": "8192",
+        "portMappings": [
+          {
+            "name": "xuexiao-5050-tcp",
+            "containerPort": 5050,
+            "hostPort": 5050,
+            "protocol": "tcp",
+            "appProtocol": "http"
+          }
+        ],
+        "command": [
+          "uvicorn",
+          "main:app",
+          "--host",
+          "0.0.0.0",
+          "--port",
+          "5050",
+          "--workers",
+          "6"
+        ],
+        "essential": true,
+        "environment": [],
+        "environmentFiles": [
+          {
+            "value": "arn:aws:s3:::xuexiao-env-variables/preview.env",
+            "type": "s3"
+          }
+        ],
+        "mountPoints": [],
+        "volumesFrom": [],
+        "ulimits": [],
+        "logConfiguration": {
+          "logDriver": "awslogs",
+          "options": {
+            "awslogs-create-group": "true",
+            "awslogs-group": "/ecs/xuexiao-preview",
+            "awslogs-region": "ap-northeast-1",
+            "awslogs-stream-prefix": "ecs"
+          }
+        }
+      }
+    ],
+    "family": "quivr-preview",
+    "taskRoleArn": "arn:aws:iam::654654285942:role/ecsTaskExecutionRole",
+    "executionRoleArn": "arn:aws:iam::654654285942:role/ecsTaskExecutionRole",
+    "networkMode": "awsvpc",
+    "revision": 4,
+    "volumes": [],
+    "status": "ACTIVE",
+    "requiresAttributes": [
+      {
+        "name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
+      },
+      {
+        "name": "ecs.capability.execution-role-awslogs"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.ecr-auth"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
+      },
+      {
+        "name": "ecs.capability.env-files.s3"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.task-iam-role"
+      },
+      {
+        "name": "ecs.capability.execution-role-ecr-pull"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
+      },
+      {
+        "name": "ecs.capability.task-eni"
+      },
+      {
+        "name": "com.amazonaws.ecs.capability.docker-remote-api.1.29"
+      }
+    ],
+    "placementConstraints": [],
+    "compatibilities": ["EC2", "FARGATE"],
+    "requiresCompatibilities": ["FARGATE"],
+    "cpu": "2048",
+    "memory": "8192",
+    "runtimePlatform": {
+      "cpuArchitecture": "X86_64",
+      "operatingSystemFamily": "LINUX"
+    },
+    "registeredAt": "2023-06-29T07:33:39.537Z",
+    "registeredBy": "arn:aws:iam::654654285942:root",
+    "tags": []
+  }
diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml
@@ -0,0 +1,110 @@
+name: Deploy Backend to Preview ECS
+
+on:
+  push:
+    branches: ["main"]
+    paths:
+      - "server/**"
+
+env:
+  AWS_REGION: ap-northeast-1
+  ECR_REPOSITORY: xuexiao
+  ECR_REGISTRY: 654654285942.dkr.ecr.ap-northeast-1.amazonaws.com
+  ECS_CLUSTER: xuexiao
+
+jobs:
+  build_and_push:
+    name: Build and Push Docker Image
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login to Docker Hub
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3
+
+      - name: Create Docker Cacha Storage Backend
+        run: |
+          docker buildx create --use --driver=docker-container
+      - name: See the file in the runner
+        run: |
+          ls -la
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          IMAGE_TAG: ${{ github.sha }}
+        with:
+          context: ./server/
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}, ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  deploy:
+    needs: build_and_push
+    runs-on: ubuntu-latest
+    environment: production
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: "xuexiao"
+            service: "xuexiao-preview"
+            task_definition: ".aws/task_definition_preview.json"
+            container: "xuexiao"
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Fill in the new image ID in the Amazon ECS task definition for ${{ matrix.name }}
+        id: task-def
+        uses: aws-actions/amazon-ecs-render-task-definition@4225e0b507142a2e432b018bc3ccb728559b437a # v1
+        with:
+          task-definition: ${{ matrix.task_definition }}
+          container-name: ${{ matrix.container }}
+          image: ${{env.ECR_REGISTRY}}/${{ env.ECR_REPOSITORY }}:${{ github.sha }}
+
+      - name: Deploy Amazon ECS task definition for ${{ matrix.name }}
+        uses: aws-actions/amazon-ecs-deploy-task-definition@df9643053eda01f169e64a0e60233aacca83799a # v1
+        with:
+          task-definition: ${{ steps.task-def.outputs.task-definition }}
+          service: ${{ matrix.service }}
+          cluster: ${{ env.ECS_CLUSTER }}
+          wait-for-service-stability: true
diff --git a/bin/act b/bin/act
diff --git a/doc/deploy_local.md b/doc/deploy_local.md
@@ -15,7 +15,7 @@ Follow these steps to start BotMeta locally:
 git clone --depth 1 [email protected]:ant-xuexiao/bot-meta.git
 
 # Copy the fake env vars
-cp server/.env.example server/.env
+cp docker/.env.example docker/.env
 cp .env.example .env
 
 # Start the services (in detached mode)

diff --git a/doc/init_aws.md b/doc/init_aws.md
@@ -13,15 +13,15 @@ https://ap-northeast-1.signin.aws/platform/login
 #### Docker Login 
 
 ```bash
-aws ecr get-login-password \
+$ aws ecr get-login-password \
         --region ap-northeast-1 | docker login \
         --username AWS \
         --password-stdin 654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao
 ```
 
 #### Build Docker Image
 ```bash
-docker build -t bot-meta .
+$ docker build -t bot-meta .
 ```
 
 > If successful, the command line will display the following information:
@@ -51,9 +51,47 @@ docker build -t bot-meta .
 #### Tag and Push Docker Image
 
 ```bash
-docker tag bot-meta:latest 654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao:latest
+$ docker tag bot-meta:latest 654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao:latest
 ```
 
 ```
-docker push 654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao:latest
+$ docker push 654654285942.dkr.ecr.ap-northeast-1.amazonaws.com/xuexiao:latest
+```
+
+
+#### Create an ECS context
+```bash
+$ docker context create ecs botmetaecscontext
+```
+
+```
+Docker Compose's integration for ECS and ACI will be retired in November 2023. Learn more: https://docs.docker.com/go/compose-ecs-eol/
+? Create a Docker context using: An existing AWS profile
+? Select AWS Profile 654654285942_administratoraccess
+Successfully created ecs context "botmetaecscontext"
+```
+
+The current context in use is marked by  * in the output of context listing:
+
+```bash
+$ docker context ls
+```
+
+```
+NAME                TYPE                DESCRIPTION                               DOCKER ENDPOINT                                KUBERNETES ENDPOINT   ORCHESTRATOR
+botmetaecscontext   ecs
+default             moby                Current DOCKER_HOST based configuration   unix:///var/run/docker.sock
+```
+
+To make all subsequent commands target Amazon ECS, make the newly created ECS context the one in use by running:
+
+```bash
+docker context use botmetaecscontext
+```
+
+
+Docker Compose converts the Compose file to a CloudFormation template defining a set of AWS resources. Details on the resource mapping can be found in the documentation. To review the CloudFormation template generated, we can run the command:
+
+```bash
+$ docker compose convert
 ```