From ecb4b604c8fc19843c898226978cd8868782215c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 25 Mar 2024 10:54:11 -0400 Subject: [PATCH 1/5] chore: rc version Signed-off-by: Evan Prodromou --- app/package-lock.json | 4 ++-- app/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/package-lock.json b/app/package-lock.json index fdb0bd24e..b6761e653 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -1,12 +1,12 @@ { "name": "city-catalyst", - "version": "0.5.0-dev.0", + "version": "0.5.0-rc.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "city-catalyst", - "version": "0.5.0-dev.0", + "version": "0.5.0-rc.0", "dependencies": { "@chakra-ui/icons": "^2.1.0", "@chakra-ui/next-js": "^2.2.0", diff --git a/app/package.json b/app/package.json index 52604d4b0..40e18a951 100644 --- a/app/package.json +++ b/app/package.json @@ -1,6 +1,6 @@ { "name": "city-catalyst", - "version": "0.5.0-dev.0", + "version": "0.5.0-rc.0", "private": true, "type": "module", "scripts": { From daf9f43c2322025f0457473efa40373786e2f987 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 25 Mar 2024 10:38:32 -0400 Subject: [PATCH 2/5] feat: automatically deploy test Signed-off-by: Evan Prodromou --- ...api-actions.yml => global-api-develop.yml} | 2 +- .github/workflows/global-api-test.yml | 59 ++++++++ .../{web-actions.yml => web-develop.yml} | 0 .github/workflows/web-test.yml | 139 ++++++++++++++++++ k8s/test/cc-test-create-admin.yml | 29 ++++ k8s/test/cc-test-global-api-deploy.yml | 2 +- k8s/test/cc-test-global-api-migrate.yml | 2 +- k8s/test/cc-test-migrate.yml | 2 +- k8s/test/cc-test-seed.yml | 29 ++++ k8s/test/cc-test-sync-catalogue.yml | 33 +++++ k8s/test/cc-test-web-deploy.yml | 2 +- 11 files changed, 294 insertions(+), 5 deletions(-) rename .github/workflows/{global-api-actions.yml => global-api-develop.yml} (97%) create mode 100644 .github/workflows/global-api-test.yml rename .github/workflows/{web-actions.yml => web-develop.yml} (100%) create mode 100644 .github/workflows/web-test.yml create mode 100644 k8s/test/cc-test-create-admin.yml create mode 100644 k8s/test/cc-test-seed.yml create mode 100644 k8s/test/cc-test-sync-catalogue.yml diff --git a/.github/workflows/global-api-actions.yml b/.github/workflows/global-api-develop.yml similarity index 97% rename from .github/workflows/global-api-actions.yml rename to .github/workflows/global-api-develop.yml index 4bd7b49d3..e5aea87b6 100644 --- a/.github/workflows/global-api-actions.yml +++ b/.github/workflows/global-api-develop.yml @@ -1,4 +1,4 @@ -name: Push Global API to GHCR and deploy to EKS +name: Push Global API to latest and deploy to EKS on: workflow_dispatch: diff --git a/.github/workflows/global-api-test.yml b/.github/workflows/global-api-test.yml new file mode 100644 index 000000000..09369e351 --- /dev/null +++ b/.github/workflows/global-api-test.yml @@ -0,0 +1,59 @@ +name: Push Global API to GHCR and deploy to EKS + +on: + workflow_dispatch: + + push: + paths: + - global-api/** + - k8s/test/cc-test-global-api-deploy.yml + - k8s/test/cc-test-global-api.yml + - k8s/test/cc-test-global-api-migrate.yml + - .github/workflows/global-api-test.yml + branches: ["main"] + +jobs: + + pushToGHCR: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pushing Global API to GHCR + env: + VERSION: ${{ github.sha }} + IMAGE: ghcr.io/open-earth-foundation/citycatalyst-global-api + run: | + docker build -t $IMAGE:$VERSION global-api + docker tag $IMAGE:$VERSION $IMAGE:main + docker push $IMAGE:$VERSION + docker push $IMAGE:main + + deployToEKS: + needs: pushToGHCR + runs-on: ubuntu-latest + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_EKS_DEV_USER }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_EKS_DEV_USER }} + EKS_DEV_NAME: ${{ secrets.EKS_DEV_NAME }} + steps: + - uses: actions/checkout@v4 + + - name: Creating kubeconfig file + run: aws eks update-kubeconfig --name ${{secrets.EKS_DEV_NAME}} --region us-east-1 + + - name: Testing connection to EKS + run: kubectl get pods -n default + + - name: Deploying service + run: | + kubectl create -f k8s/cc-test-global-api-migrate.yml -n default + kubectl apply -f k8s/cc-test-global-api-deploy.yml -n default + kubectl rollout restart deployment cc-test-global-api-deploy -n default diff --git a/.github/workflows/web-actions.yml b/.github/workflows/web-develop.yml similarity index 100% rename from .github/workflows/web-actions.yml rename to .github/workflows/web-develop.yml diff --git a/.github/workflows/web-test.yml b/.github/workflows/web-test.yml new file mode 100644 index 000000000..70638bf7c --- /dev/null +++ b/.github/workflows/web-test.yml @@ -0,0 +1,139 @@ +name: Push Web app to GHCR and deploy to EKS + +on: + workflow_dispatch: + push: + paths: + - app/** + - k8s/test/cc-test-migrate.yml + - k8s/test/cc-test-web-deploy.yml + - k8s/test/cc-test/web.yml + - .github/workflows/web-test.yml + branches: ["develop"] + pull_request: + +jobs: + runTests: + runs-on: ubuntu-latest + env: + NODE_ENV: test + NEXTAUTH_SECRET: "diTMz/XLX4edSmmfzwJtmzKjCJGRt81Gf0PdjO3IPs8=" + NEXTAUTH_URL: "http://localhost:3000" + defaults: + run: + working-directory: ./app + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: "npm" + cache-dependency-path: app/package-lock.json + + - name: Install dependencies + run: npm ci + + - name: Set up database + run: | + docker run --name github_action_postgresql -d -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -e POSTGRES_PASSWORD="" postgres + sleep 10 + createuser -w -h localhost -p 5432 -U postgres citycatalyst + createdb -w -h localhost -p 5432 -U postgres citycatalyst -O citycatalyst + cp env.example .env + npm run db:migrate + + - name: Run NextJS build + run: npm run build + + # - name: Run Cypress tests + # run: npm run cy:test + + # ... Generate LCOV files or download it from a different job + - name: Run tests and generate coverage file + run: npm run ci:test + + # - name: Setup LCOV + # uses: hrishikesh-kadam/setup-lcov@v1 + # - name: Report code coverage + # uses: zgosalvez/github-actions-report-lcov@v3 + # with: + # coverage-files: ./app/lcov*.info + # minimum-coverage: 40 + # artifact-name: code-coverage-report + # github-token: ${{ secrets.GITHUB_TOKEN }} + # working-directory: ./app + # update-comment: true + + - name: Upload coverage reports to Codecov + continue-on-error: true + uses: codecov/codecov-action@v4.0.1 + with: + fail_ci_if_error: false + flags: unittests + name: citycatalyst-backend + token: ${{ secrets.CODECOV_TOKEN }} + slug: Open-Earth-Foundation/CityCatalyst + + - name: Shut down database + run: docker stop github_action_postgresql + + pushToGHCR: + needs: runTests + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/develop' + steps: + - uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pushing CC Web to GHCR + env: + VERSION: ${{ github.sha }} + IMAGE: ghcr.io/open-earth-foundation/citycatalyst + run: | + docker build -t $IMAGE:$VERSION app + docker tag $IMAGE:$VERSION $IMAGE:main + docker push $IMAGE:$VERSION + docker push $IMAGE:main + + deployToEKS: + needs: pushToGHCR + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_EKS_DEV_USER }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_EKS_DEV_USER }} + EKS_DEV_NAME: ${{ secrets.EKS_DEV_NAME }} + steps: + - uses: actions/checkout@v4 + + - name: Creating kubeconfig file + run: aws eks update-kubeconfig --name ${{secrets.EKS_DEV_NAME}} --region us-east-1 + + - name: Testing connection to EKS + run: kubectl get pods -n default + + - name: Deploying service + run: | + kubectl create -f k8s/test/cc-test-migrate.yml -n default + kubectl create -f k8s/test/cc-test-seed.yml -n default + kubectl apply -f k8s/test/cc-test-sync-catalogue.yml -n default + kubectl apply -f k8s/test/cc-test-web-deploy.yml -n default + kubectl set env deployment/cc-test-web-deploy SMTP_USER=${{secrets.SMTP_USER}} + kubectl set env deployment/cc-test-web-deploy SMTP_PASSWORD=${{secrets.SMTP_PASSWORD}} + kubectl set env deployment/cc-test-web-deploy NEXTAUTH_SECRET=${{secrets.NEXTAUTH_SECRET}} + kubectl set env deployment/cc-test-web-deploy RESET_TOKEN_SECRET=${{secrets.RESET_TOKEN_SECRET}} + kubectl set env deployment/cc-test-web-deploy VERIFICATION_TOKEN_SECRET=${{secrets.VERIFICATION_TOKEN_SECRET}} + kubectl set env deployment/cc-test-web-deploy CHAT_PROVIDER=openai + kubectl set env deployment/cc-test-web-deploy OPENAI_API_KEY=${{secrets.OPENAI_API_KEY}} + kubectl set env deployment/cc-test-web-deploy HUGGINGFACE_API_KEY=${{secrets.HUGGINGFACE_API_KEY}} + kubectl set env deployment/cc-test-web-deploy "DEFAULT_ADMIN_EMAIL=${{secrets.DEFAULT_ADMIN_EMAIL}}" + kubectl set env deployment/cc-test-web-deploy "DEFAULT_ADMIN_PASSWORD=${{secrets.DEFAULT_ADMIN_PASSWORD}}" + kubectl create -f k8s/test/cc-test-create-admin.yml -n default + kubectl rollout restart deployment cc-test-web-deploy -n default diff --git a/k8s/test/cc-test-create-admin.yml b/k8s/test/cc-test-create-admin.yml new file mode 100644 index 000000000..7763e898e --- /dev/null +++ b/k8s/test/cc-test-create-admin.yml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: cc-test-create-admin- +spec: + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: cc-test-create-admin + image: ghcr.io/open-earth-foundation/citycatalyst:main + imagePullPolicy: Always + env: + - name: NODE_ENV + value: development + - name: DATABASE_NAME + value: "citycatalyst" + - name: DATABASE_HOST + value: "cc-db" + - name: DATABASE_USER + value: "citycatalyst" + - name: DATABASE_PASSWORD + value: "development" + command: ["npm", "run", "create-admin"] + resources: + limits: + memory: "1024Mi" + cpu: "1000m" diff --git a/k8s/test/cc-test-global-api-deploy.yml b/k8s/test/cc-test-global-api-deploy.yml index 779672b4d..1bafd997c 100644 --- a/k8s/test/cc-test-global-api-deploy.yml +++ b/k8s/test/cc-test-global-api-deploy.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: cc-test-global-api - image: ghcr.io/open-earth-foundation/citycatalyst-global-api:latest + image: ghcr.io/open-earth-foundation/citycatalyst-global-api:main # Set to Never for local imagePullPolicy: Always ports: diff --git a/k8s/test/cc-test-global-api-migrate.yml b/k8s/test/cc-test-global-api-migrate.yml index 3aca58920..fdb899c35 100644 --- a/k8s/test/cc-test-global-api-migrate.yml +++ b/k8s/test/cc-test-global-api-migrate.yml @@ -9,7 +9,7 @@ spec: restartPolicy: OnFailure containers: - name: cc-test-global-api - image: ghcr.io/open-earth-foundation/citycatalyst-global-api:latest + image: ghcr.io/open-earth-foundation/citycatalyst-global-api:main env: - name: ALEMBIC_URL value: "postgresql://ccglobaltest:ccglobaltest@cc-global-api-db/ccglobaltest" diff --git a/k8s/test/cc-test-migrate.yml b/k8s/test/cc-test-migrate.yml index fe45d433a..2ecd41a4d 100644 --- a/k8s/test/cc-test-migrate.yml +++ b/k8s/test/cc-test-migrate.yml @@ -9,7 +9,7 @@ spec: restartPolicy: OnFailure containers: - name: cc-test-migrate - image: ghcr.io/open-earth-foundation/citycatalyst:latest + image: ghcr.io/open-earth-foundation/citycatalyst:main imagePullPolicy: Always env: - name: NODE_ENV diff --git a/k8s/test/cc-test-seed.yml b/k8s/test/cc-test-seed.yml new file mode 100644 index 000000000..1f1f35418 --- /dev/null +++ b/k8s/test/cc-test-seed.yml @@ -0,0 +1,29 @@ +apiVersion: batch/v1 +kind: Job +metadata: + generateName: cc-test-seed- +spec: + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: cc-test-seed + image: ghcr.io/open-earth-foundation/citycatalyst:main + imagePullPolicy: Always + env: + - name: NODE_ENV + value: development + - name: DATABASE_NAME + value: "citycatalyst" + - name: DATABASE_HOST + value: "cc-db" + - name: DATABASE_USER + value: "citycatalyst" + - name: DATABASE_PASSWORD + value: "development" + command: ["npm", "run", "db:seed"] + resources: + limits: + memory: "1024Mi" + cpu: "1000m" diff --git a/k8s/test/cc-test-sync-catalogue.yml b/k8s/test/cc-test-sync-catalogue.yml new file mode 100644 index 000000000..5dc1e9186 --- /dev/null +++ b/k8s/test/cc-test-sync-catalogue.yml @@ -0,0 +1,33 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cc-test-sync-catalogue +spec: + # run every monday + schedule: 0 0 * * 1 + jobTemplate: + spec: + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: OnFailure + containers: + - name: cc-test-sync-catalogue + image: ghcr.io/open-earth-foundation/citycatalyst:main + imagePullPolicy: Always + env: + - name: NODE_ENV + value: development + - name: DATABASE_NAME + value: "citycatalyst" + - name: DATABASE_HOST + value: "cc-db" + - name: DATABASE_USER + value: "citycatalyst" + - name: DATABASE_PASSWORD + value: "development" + command: ["npx", "tsx", "sync-catalogue"] + resources: + limits: + memory: "1024Mi" + cpu: "1000m" diff --git a/k8s/test/cc-test-web-deploy.yml b/k8s/test/cc-test-web-deploy.yml index c2ec058e3..8280f3886 100644 --- a/k8s/test/cc-test-web-deploy.yml +++ b/k8s/test/cc-test-web-deploy.yml @@ -16,7 +16,7 @@ spec: spec: containers: - name: cc-test-web - image: ghcr.io/open-earth-foundation/citycatalyst:latest + image: ghcr.io/open-earth-foundation/citycatalyst:main # Set to Never for local imagePullPolicy: Always ports: From dc16d19ff7a835ef5d6548340ff2552e344f91aa Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 25 Mar 2024 10:42:06 -0400 Subject: [PATCH 3/5] fix: deploy to test on main --- .github/workflows/web-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web-test.yml b/.github/workflows/web-test.yml index 70638bf7c..6fb912cdb 100644 --- a/.github/workflows/web-test.yml +++ b/.github/workflows/web-test.yml @@ -9,7 +9,7 @@ on: - k8s/test/cc-test-web-deploy.yml - k8s/test/cc-test/web.yml - .github/workflows/web-test.yml - branches: ["develop"] + branches: ["main"] pull_request: jobs: From 9dcd8cbe596cd64f69e2b96e44770a3010e93798 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 25 Mar 2024 12:35:38 -0400 Subject: [PATCH 4/5] fix: correct path in global api deployment scripts Signed-off-by: Evan Prodromou --- .github/workflows/global-api-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/global-api-test.yml b/.github/workflows/global-api-test.yml index 09369e351..17a46df8e 100644 --- a/.github/workflows/global-api-test.yml +++ b/.github/workflows/global-api-test.yml @@ -54,6 +54,6 @@ jobs: - name: Deploying service run: | - kubectl create -f k8s/cc-test-global-api-migrate.yml -n default - kubectl apply -f k8s/cc-test-global-api-deploy.yml -n default + kubectl create -f k8s/test/cc-test-global-api-migrate.yml -n default + kubectl apply -f k8s/test/cc-test-global-api-deploy.yml -n default kubectl rollout restart deployment cc-test-global-api-deploy -n default From bf58ea21c105263b20ea2c318606648e19cddbd8 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 25 Mar 2024 12:37:49 -0400 Subject: [PATCH 5/5] fix: don't run test deployment on pull request Signed-off-by: Evan Prodromou --- .github/workflows/web-test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/web-test.yml b/.github/workflows/web-test.yml index 6fb912cdb..33081afe1 100644 --- a/.github/workflows/web-test.yml +++ b/.github/workflows/web-test.yml @@ -10,7 +10,6 @@ on: - k8s/test/cc-test/web.yml - .github/workflows/web-test.yml branches: ["main"] - pull_request: jobs: runTests: @@ -105,7 +104,6 @@ jobs: deployToEKS: needs: pushToGHCR runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_EKS_DEV_USER }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_EKS_DEV_USER }}