diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0eafca0e..57161757 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,16 +1,12 @@ name: Publish -on: [push] +on: [ push ] jobs: main: - name: Publish + name: Validate STAC runs-on: ubuntu-latest - concurrency: publish-${{ github.ref }} - - permissions: - id-token: write - contents: read + concurrency: validate-${{ github.ref }} steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 @@ -39,7 +35,7 @@ jobs: run: docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color deploy-prod: - name: Publish STAC + name: Publish ODR & Sync STAC runs-on: ubuntu-latest concurrency: publish-${{ github.ref }} needs: main @@ -55,14 +51,23 @@ jobs: steps: - uses: linz/action-typescript@dee99184c4305aea6c380a52db9b2d7abaaa3e78 # v3 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + with: + fetch-depth: 0 + + - name: Use Node.js 18.x + uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65 # v4.0.0 + with: + node-version: '18.x' + # FIXME: catalog.json is not pushed to the repository (temporary solution) - name: Create STAC Catalog - uses: docker://ghcr.io/linz/argo-tasks:v2 + uses: docker://ghcr.io/linz/argo-tasks:v3 with: args: stac-catalog --output stac/catalog.json --template template/catalog.json /github/workspace/stac/ - name: Validate STAC Catalog - uses: docker://ghcr.io/linz/argo-tasks:v2 + uses: docker://ghcr.io/linz/argo-tasks:v3 with: args: stac-validate /github/workspace/stac/catalog.json @@ -70,7 +75,45 @@ jobs: run: | # Enable double star operator shopt -s globstar - docker run -v "${PWD}:${PWD}" ghcr.io/linz/argo-tasks:v2 stac-validate "$PWD"/stac/**/collection.json + docker run -v "${PWD}:${PWD}" ghcr.io/linz/argo-tasks:v3 stac-validate "$PWD"/stac/**/collection.json + + - name: Setup kubectl + uses: azure/setup-kubectl@901a10e89ea615cf61f57ac05cecdf23e7de06d8 # v3 + with: + version: 'latest' + + - name: AWS Configure + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4 + with: + aws-region: ap-southeast-2 + mask-aws-account-id: true + role-to-assume: ${{ secrets.AWS_EKS_CI_ROLE }} + + - name: Login to EKS + run: | + aws eks update-kubeconfig --name Workflows --region ap-southeast-2 + + - name: Check EKS connection + run: | + kubectl get nodes + + - name: Install Argo + run: | + curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v3.4.11/argo-linux-amd64.gz + gunzip argo-linux-amd64.gz + chmod +x argo-linux-amd64 + ./argo-linux-amd64 version + + - name: Submit Added/Changed Parameter Files + id: modified-files + run: | + # AM = Include: Added, Modified + mapfile -d '' modified_parameter_files < <(git diff --name-only --diff-filter=AM -z ${{ github.event.before }} ${{ github.event.after }} -- "publish-odr-parameters/*.yaml") + + for file in "${modified_parameter_files[@]}"; do + base=$(basename "${file%.yaml}") + ./argo-linux-amd64 submit --wait --from wftmpl/copy -n argo -f "$file" -p aws_role_config_path="s3://linz-bucket-config/config-write.open-data-registry.json" --generate-name "publish-odr-$base-" + done - name: AWS Configure uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4 @@ -87,7 +130,8 @@ jobs: role-to-assume: ${{ secrets.AWS_ODR_DATA_MANAGER_ROLE }} role-chaining: true + # Sync STAC files only on push to 'master' - name: Sync STAC - uses: docker://ghcr.io/linz/argo-tasks:v2 + uses: docker://ghcr.io/linz/argo-tasks:v3 with: args: stac-sync /github/workspace/stac/ s3://nz-elevation/ diff --git a/publish-odr-parameters/README.md b/publish-odr-parameters/README.md new file mode 100644 index 00000000..71b0315b --- /dev/null +++ b/publish-odr-parameters/README.md @@ -0,0 +1,36 @@ +# ODR Copy Workflow Parameters + +This folder contains the parameter yaml files generated by the `publish-odr` argo workflow. + +## How it Works: + +### Step 1: Submit the publish-odr workflow using Argo + +See [publish-odr](https://github.com/linz/topo-workflows/blob/master/workflows/raster/publish-odr.yaml) for more details. + +### Step 2: Review and Approve + +- Review and approve the STAC metadata and basemaps config preview link. +- Review the target path in the parameters yaml file. + +Example parameters file: + +```yaml +source: 's3://path/to/source/' +target: 's3://path/to/target/' +``` + +> [!Tip] +> If the parameters are incorrect or additional parameters are required they can be updated/added here. +> See the [copy workflow](https://github.com/linz/topo-workflows/tree/master/workflows/raster#copy) for additonal parameter options & default values. + +### Step 3: Merge Pull Request + +Once the pull request has been merged the [publish.yml](../.github/workflows/publish.yml) Github Action will run. + +This will: + +- Create a `catalog.json`. +- Validate the STAC. +- Submit a `copy` argo workflow for any added or modified parameter files in the `./publish-odr-parameters/` folder. +- Sync the `catalog.json` & `collection.json` created in this PR with the ODR bucket.