ci: add argo copy steps

linz · Mar 19, 2024 · 5dda664 · 5dda664
1 parent 4ca37ff
commit 5dda664
Show file tree

Hide file tree

Showing 2 changed files with 92 additions and 12 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -1,16 +1,12 @@
 name: Publish
 
-on: [push]
+on: [ push ]
 
 jobs:
   main:
-    name: Publish
+    name: Validate STAC
     runs-on: ubuntu-latest
-    concurrency: publish-${{ github.ref }}
-
-    permissions:
-      id-token: write
-      contents: read
+    concurrency: validate-${{ github.ref }}
 
     steps:
       - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3
@@ -39,7 +35,7 @@ jobs:
         run: docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color
 
   deploy-prod:
-    name: Publish STAC
+    name: Publish ODR & Sync STAC
     runs-on: ubuntu-latest
     concurrency: publish-${{ github.ref }}
     needs: main
@@ -55,22 +51,69 @@ jobs:
     steps:
       - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3
 
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+        with:
+          node-version: '18.x'
+
       # FIXME: catalog.json is not pushed to the repository (temporary solution)
       - name: Create STAC Catalog
-        uses: docker://ghcr.io/linz/argo-tasks:v2
+        uses: docker://ghcr.io/linz/argo-tasks:v3
         with:
           args: stac-catalog --output stac/catalog.json --template template/catalog.json /github/workspace/stac/
 
       - name: Validate STAC Catalog
-        uses: docker://ghcr.io/linz/argo-tasks:v2
+        uses: docker://ghcr.io/linz/argo-tasks:v3
         with:
           args: stac-validate /github/workspace/stac/catalog.json
 
       - name: Validate STAC Collections
         run: |
           # Enable double star operator
           shopt -s globstar
-          docker run -v "${PWD}:${PWD}" ghcr.io/linz/argo-tasks:v2 stac-validate "$PWD"/stac/**/collection.json
+          docker run -v "${PWD}:${PWD}" ghcr.io/linz/argo-tasks:v3 stac-validate "$PWD"/stac/**/collection.json
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@901a10e89ea615cf61f57ac05cecdf23e7de06d8 # v3
+        with:
+          version: 'latest'
+
+      - name: AWS Configure
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4
+        with:
+          aws-region: ap-southeast-2
+          mask-aws-account-id: true
+          role-to-assume: ${{ secrets.AWS_EKS_CI_ROLE }}
+
+      - name: Login to EKS
+        run: |
+          aws eks update-kubeconfig --name Workflows --region ap-southeast-2
+
+      - name: Check EKS connection
+        run: |
+          kubectl get nodes
+
+      - name: Install Argo
+        run: |
+          curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.11/argo-linux-amd64.gz
+          gunzip argo-linux-amd64.gz
+          chmod +x argo-linux-amd64
+          ./argo-linux-amd64 version
+
+      - name: Submit Added/Changed Parameter Files
+        id: modified-files
+        run: |
+          # AM = Include: Added, Modified
+          mapfile -d '' modified_parameter_files < <(git diff --name-only --diff-filter=AM -z ${{ github.event.before }} ${{ github.event.after }} -- "publish-odr-parameters/*.yaml")
+
+          for file in "${modified_parameter_files[@]}"; do
+            base=$(basename "${file%.yaml}")
+            ./argo-linux-amd64 submit --wait --from wftmpl/copy -n argo -f "$file" -p aws_role_config_path="s3://linz-bucket-config/config-write.open-data-registry.json" --generate-name "publish-odr-$base-"
+          done
 
       - name: AWS Configure
         uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4
@@ -87,7 +130,8 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ODR_DATA_MANAGER_ROLE }}
           role-chaining: true
 
+      # Sync STAC files only on push to 'master'
       - name: Sync STAC
-        uses: docker://ghcr.io/linz/argo-tasks:v2
+        uses: docker://ghcr.io/linz/argo-tasks:v3
         with:
           args: stac-sync /github/workspace/stac/ s3://nz-elevation/
diff --git a/publish-odr-parameters/README.md b/publish-odr-parameters/README.md
@@ -0,0 +1,36 @@
+# ODR Copy Workflow Parameters
+
+This folder contains the parameter yaml files generated by the `publish-odr` argo workflow.
+
+## How it Works:
+
+### Step 1: Submit the publish-odr workflow using Argo
+
+See [publish-odr](https://github.com/linz/topo-workflows/blob/master/workflows/raster/publish-odr.yaml) for more details.
+
+### Step 2: Review and Approve
+
+- Review and approve the STAC metadata and basemaps config preview link.
+- Review the target path in the parameters yaml file.
+
+Example parameters file:
+
+```yaml
+source: 's3://path/to/source/'
+target: 's3://path/to/target/'
+```
+
+> [!Tip]
+> If the parameters are incorrect or additional parameters are required they can be updated/added here.
+> See the [copy workflow](https://github.com/linz/topo-workflows/tree/master/workflows/raster#copy) for additonal parameter options & default values.
+
+### Step 3: Merge Pull Request
+
+Once the pull request has been merged the [publish.yml](../.github/workflows/publish.yml) Github Action will run.
+
+This will:
+
+- Create a `catalog.json`.
+- Validate the STAC.
+- Submit a `copy` argo workflow for any added or modified parameter files in the `./publish-odr-parameters/` folder.
+- Sync the `catalog.json` & `collection.json` created in this PR with the ODR bucket.