ci: add argo copy steps TDE-1099 (#281)

note: this code has been copied across from
[linz/imagery/.github/workflows/publish.yaml](https://github.com/linz/imagery/blob/master/.github/workflows/publish.yml)

The intention of this change is to add the argo steps in order to
support the automation of publishing to the ODR.
However, there are some other changes as well (like renaming and version
updates) in order to keep the two publish workflows almost identical -
making any future updates easier.

depends on: linz/topo-aws-infrastructure#336

.github/workflows/publish.yml

@@ -1,16 +1,12 @@
 name: Publish
 
-on: [push]
+on: [ push ]
 
 jobs:
   main:
-    name: Publish
+    name: Validate STAC
     runs-on: ubuntu-latest
-    concurrency: publish-${{ github.ref }}
-
-    permissions:
-      id-token: write
-      contents: read
+    concurrency: validate-${{ github.ref }}
 
     steps:
       - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3
@@ -39,7 +35,7 @@ jobs:
         run: docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color
 
   deploy-prod:
-    name: Publish STAC
+    name: Publish ODR & Sync STAC
     runs-on: ubuntu-latest
     concurrency: publish-${{ github.ref }}
     needs: main
@@ -55,22 +51,69 @@ jobs:
     steps:
       - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3
 
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+        with:
+          fetch-depth: 0
+
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0
+        with:
+          node-version: '18.x'
+
       # FIXME: catalog.json is not pushed to the repository (temporary solution)
       - name: Create STAC Catalog
-        uses: docker://ghcr.io/linz/argo-tasks:v2
+        uses: docker://ghcr.io/linz/argo-tasks:v3
         with:
           args: stac-catalog --output stac/catalog.json --template template/catalog.json /github/workspace/stac/
 
       - name: Validate STAC Catalog
-        uses: docker://ghcr.io/linz/argo-tasks:v2
+        uses: docker://ghcr.io/linz/argo-tasks:v3
         with:
           args: stac-validate /github/workspace/stac/catalog.json
 
       - name: Validate STAC Collections
         run: |
           # Enable double star operator
           shopt -s globstar
-          docker run -v "${PWD}:${PWD}" ghcr.io/linz/argo-tasks:v2 stac-validate "$PWD"/stac/**/collection.json
+          docker run -v "${PWD}:${PWD}" ghcr.io/linz/argo-tasks:v3 stac-validate "$PWD"/stac/**/collection.json
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@901a10e89ea615cf61f57ac05cecdf23e7de06d8 # v3
+        with:
+          version: 'latest'
+
+      - name: AWS Configure
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4
+        with:
+          aws-region: ap-southeast-2
+          mask-aws-account-id: true
+          role-to-assume: ${{ secrets.AWS_EKS_CI_ROLE }}
+
+      - name: Login to EKS
+        run: |
+          aws eks update-kubeconfig --name Workflows --region ap-southeast-2
+
+      - name: Check EKS connection
+        run: |
+          kubectl get nodes
+
+      - name: Install Argo
+        run: |
+          curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.11/argo-linux-amd64.gz
+          gunzip argo-linux-amd64.gz
+          chmod +x argo-linux-amd64
+          ./argo-linux-amd64 version
+
+      - name: Submit Added/Changed Parameter Files
+        id: modified-files
+        run: |
+          # AM = Include: Added, Modified
+          mapfile -d '' modified_parameter_files < <(git diff --name-only --diff-filter=AM -z ${{ github.event.before }} ${{ github.event.after }} -- "publish-odr-parameters/*.yaml")
+
+          for file in "${modified_parameter_files[@]}"; do
+            base=$(basename "${file%.yaml}")
+            ./argo-linux-amd64 submit --wait --from wftmpl/copy -n argo -f "$file" -p aws_role_config_path="s3://linz-bucket-config/config-write.open-data-registry.json" --generate-name "publish-odr-$base-"
+          done
 
       - name: AWS Configure
         uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4
@@ -87,7 +130,8 @@ jobs:
           role-to-assume: ${{ secrets.AWS_ODR_DATA_MANAGER_ROLE }}
           role-chaining: true
 
+      # Sync STAC files only on push to 'master'
       - name: Sync STAC
-        uses: docker://ghcr.io/linz/argo-tasks:v2
+        uses: docker://ghcr.io/linz/argo-tasks:v3
         with:
           args: stac-sync /github/workspace/stac/ s3://nz-elevation/

publish-odr-parameters/README.md

@@ -0,0 +1,36 @@
+# ODR Copy Workflow Parameters
+
+This folder contains the parameter yaml files generated by the `publish-odr` argo workflow.
+
+## How it Works:
+
+### Step 1: Submit the publish-odr workflow using Argo
+
+See [publish-odr](https://github.com/linz/topo-workflows/blob/master/workflows/raster/publish-odr.yaml) for more details.
+
+### Step 2: Review and Approve
+
+- Review and approve the STAC metadata and basemaps config preview link.
+- Review the target path in the parameters yaml file.
+
+Example parameters file:
+
+```yaml
+source: 's3://path/to/source/'
+target: 's3://path/to/target/'
+```
+
+> [!Tip]
+> If the parameters are incorrect or additional parameters are required they can be updated/added here.
+> See the [copy workflow](https://github.com/linz/topo-workflows/tree/master/workflows/raster#copy) for additonal parameter options & default values.
+
+### Step 3: Merge Pull Request
+
+Once the pull request has been merged the [publish.yml](../.github/workflows/publish.yml) Github Action will run.
+
+This will:
+
+- Create a `catalog.json`.
+- Validate the STAC.
+- Submit a `copy` argo workflow for any added or modified parameter files in the `./publish-odr-parameters/` folder.
+- Sync the `catalog.json` & `collection.json` created in this PR with the ODR bucket.