From b91f5a7b6b1a1cb16d6c76e0c4379a9f8877d6ea Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Sat, 15 Jul 2023 01:22:11 -0400 Subject: [PATCH 1/3] Revert "0 fetch and gather" This reverts commit 0d3bd1570d873641cda7934738ae1e4f2b2100fc. --- vars.prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vars.prod.yml b/vars.prod.yml index 5be1d9a1..6d918b29 100644 --- a/vars.prod.yml +++ b/vars.prod.yml @@ -7,8 +7,8 @@ ckanext__saml2auth__idp_metadata__local_path: ckan/setup/login.production.idp.xm web-instances: 5 admin-instances: 1 -gather-instances: 0 -fetch-instances: 0 +gather-instances: 1 +fetch-instances: 4 memory_quota: 850M gather_memory_quota: 3G From fb1db4bb402414f4b0bf9de43040f7b820daa158 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Sat, 15 Jul 2023 01:22:14 -0400 Subject: [PATCH 2/3] Revert "stop backend jobs on prod to lock DB, preparing ckan210 release" This reverts commit 061a23d557e096f8124a8cc75bbcf9df0ed9c54b. --- .github/workflows/ckan.yml | 1 + .github/workflows/ckan_auto.yml | 5 +++-- .github/workflows/publish.yml | 22 +++++++++++++++++++++- .github/workflows/restart.yml | 6 +++--- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ckan.yml b/.github/workflows/ckan.yml index 48b46880..776cd789 100644 --- a/.github/workflows/ckan.yml +++ b/.github/workflows/ckan.yml @@ -10,6 +10,7 @@ on: type: choice options: - development + - prod app: description: 'App to run on:' required: true diff --git a/.github/workflows/ckan_auto.yml b/.github/workflows/ckan_auto.yml index 70c5e104..9148f02d 100644 --- a/.github/workflows/ckan_auto.yml +++ b/.github/workflows/ckan_auto.yml @@ -58,7 +58,7 @@ jobs: "schedule": ["${{env.SCHEDULE_TRACKING}}", "${{env.SCHEDULE_SITEMAP}}", "${{env.SCHEDULE_HARVESTING}}", "${{env.SCHEDULE_STUCK_JOBS}}", "${{env.SCHEDULE_DBSOLR_SYNC}}", "${{ env.SCHEDULE_GEN_REPORT}}"], - "environ": ["development"], + "environ": ["development", "prod"], "include": [ {"app": "catalog-admin"}, {"error_seconds": 22000}, {"info_issue": false}, @@ -78,7 +78,8 @@ jobs: {"schedule": "${{env.SCHEDULE_STUCK_JOBS}}", "command": "ckan geodatagov check-stuck-jobs"}, {"schedule": "${{env.SCHEDULE_GEN_REPORT}}", "command": "ckan report generate"}, {"schedule": "${{env.SCHEDULE_GEN_REPORT}}", "issue_template": ".github/report-generation-failure.md"}, - {"environ": "development", "ram": "1G"} + {"environ": "development", "ram": "1G"}, + {"environ": "prod", "ram": "2500M"} ], "exclude": [ {"schedule": "${{env.SCHEDULE_SITEMAP}}", "environ": "development"}, {"schedule": "${{env.SCHEDULE_DBSOLR_SYNC}}", "environ": "development"}, diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 99fb7a8c..5997b611 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,6 +40,13 @@ jobs: push: true tags: ${{ matrix.tag }} + create-cloudgov-services-staging: + name: create services (staging) + uses: gsa/data.gov/.github/workflows/create-services-template.yml@main + with: + environ: staging + secrets: inherit + create-cloudgov-services-prod: name: create services (prod) uses: gsa/data.gov/.github/workflows/create-services-template.yml@main @@ -47,10 +54,22 @@ jobs: environ: prod secrets: inherit + deploy-staging: + name: deploy (staging) + needs: + - create-cloudgov-services-staging + uses: gsa/data.gov/.github/workflows/deploy-template.yml@main + with: + environ: staging + app_url: https://catalog-stage-datagov.app.cloud.gov + app_names: "{\"include\":[{\"app\":\"catalog-web\",\"smoketest\":true},{\"app\":\"catalog-admin\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-proxy\"}]}" + secrets: inherit + deploy-prod: name: deploy (prod) needs: - create-cloudgov-services-prod + - deploy-staging uses: gsa/data.gov/.github/workflows/deploy-template.yml@main with: environ: prod @@ -61,6 +80,7 @@ jobs: network-policies: name: Add network-policies needs: + - deploy-staging - deploy-prod runs-on: ubuntu-latest environment: ${{ matrix.environ }} @@ -71,7 +91,7 @@ jobs: "cf add-network-policy catalog-proxy catalog-web --protocol tcp --port 61443", "cf add-network-policy catalog-proxy catalog-admin --protocol tcp --port 61443" ] - environ: ["prod"] + environ: ["staging", "prod"] steps: - name: proxy to web uses: cloud-gov/cg-cli-tools@main diff --git a/.github/workflows/restart.yml b/.github/workflows/restart.yml index 4e4f18e5..b1d65dfe 100644 --- a/.github/workflows/restart.yml +++ b/.github/workflows/restart.yml @@ -15,10 +15,10 @@ jobs: # app_names: "{\"include\":[{\"app\":\"catalog-proxy\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-web\"},{\"app\":\"catalog-admin\"},]}" # secrets: inherit - restart-development: - name: restart (development) + restart-prod: + name: restart (prod) uses: gsa/data.gov/.github/workflows/app-restart-template.yml@main with: - environ: development + environ: prod app_names: "{\"include\":[{\"app\":\"catalog-proxy\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-web\"},{\"app\":\"catalog-admin\"},]}" secrets: inherit From a5d415c33917044fc6455c68637626d8bccc1990 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Sat, 15 Jul 2023 01:22:18 -0400 Subject: [PATCH 3/3] Revert "stop staging job, prepare for 2.10 release dryrun" This reverts commit a7f10e15b41ebdb7c30e7ef7260bb1714bc68352. --- .github/workflows/ckan.yml | 1 + .github/workflows/ckan_auto.yml | 4 +++- .github/workflows/restart.yml | 14 +++++++------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ckan.yml b/.github/workflows/ckan.yml index 776cd789..ea2e6286 100644 --- a/.github/workflows/ckan.yml +++ b/.github/workflows/ckan.yml @@ -10,6 +10,7 @@ on: type: choice options: - development + - staging - prod app: description: 'App to run on:' diff --git a/.github/workflows/ckan_auto.yml b/.github/workflows/ckan_auto.yml index 9148f02d..5458366e 100644 --- a/.github/workflows/ckan_auto.yml +++ b/.github/workflows/ckan_auto.yml @@ -58,7 +58,7 @@ jobs: "schedule": ["${{env.SCHEDULE_TRACKING}}", "${{env.SCHEDULE_SITEMAP}}", "${{env.SCHEDULE_HARVESTING}}", "${{env.SCHEDULE_STUCK_JOBS}}", "${{env.SCHEDULE_DBSOLR_SYNC}}", "${{ env.SCHEDULE_GEN_REPORT}}"], - "environ": ["development", "prod"], + "environ": ["development", "staging", "prod"], "include": [ {"app": "catalog-admin"}, {"error_seconds": 22000}, {"info_issue": false}, @@ -79,9 +79,11 @@ jobs: {"schedule": "${{env.SCHEDULE_GEN_REPORT}}", "command": "ckan report generate"}, {"schedule": "${{env.SCHEDULE_GEN_REPORT}}", "issue_template": ".github/report-generation-failure.md"}, {"environ": "development", "ram": "1G"}, + {"environ": "staging", "ram": "2500M"}, {"environ": "prod", "ram": "2500M"} ], "exclude": [ {"schedule": "${{env.SCHEDULE_SITEMAP}}", "environ": "development"}, + {"schedule": "${{env.SCHEDULE_SITEMAP}}", "environ": "staging"}, {"schedule": "${{env.SCHEDULE_DBSOLR_SYNC}}", "environ": "development"}, {"schedule": "${{env.SCHEDULE_STUCK_JOBS}}", "environ": "development"} ], diff --git a/.github/workflows/restart.yml b/.github/workflows/restart.yml index b1d65dfe..64cee9c2 100644 --- a/.github/workflows/restart.yml +++ b/.github/workflows/restart.yml @@ -7,13 +7,13 @@ on: - cron: '11/30 * * * *' jobs: - # restart-staging: - # name: restart (staging) - # uses: gsa/data.gov/.github/workflows/app-restart-template.yml@main - # with: - # environ: staging - # app_names: "{\"include\":[{\"app\":\"catalog-proxy\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-web\"},{\"app\":\"catalog-admin\"},]}" - # secrets: inherit + restart-staging: + name: restart (staging) + uses: gsa/data.gov/.github/workflows/app-restart-template.yml@main + with: + environ: staging + app_names: "{\"include\":[{\"app\":\"catalog-proxy\"},{\"app\":\"catalog-gather\"},{\"app\":\"catalog-fetch\"},{\"app\":\"catalog-web\"},{\"app\":\"catalog-admin\"},]}" + secrets: inherit restart-prod: name: restart (prod)