Cache GMT remote data as artifacts on Github (#530)

Use GMT on macOS to download the GMT remote files from the GMT data server,
and then upload them as artifacts on Github.
* Download cached remote data artifacts from Github
* Include instructions on how to manually refresh the CI data cache
* Bump action-download-artifact from 2.0.0 to 2.6.3
* Just download to `.gmt` folder and list downloaded remote files
* Move downloaded files from .gmt to ~/.gmt
* Make a directory under ~/.gmt before moving files into it

Co-authored-by: Dongdong Tian <[email protected]>

Author: weiji14

.github/workflows/cache_data.yaml

@@ -0,0 +1,42 @@
+# This workflow gets and uploads the GMT data artifacts used in the PyGMT CI tests
+name: Cache data
+
+on:
+  # Uncomment the 'pull_request' line below to manually re-cache data artifacts
+  # pull_request:
+  # Schedule runs on 12 noon every Sunday
+  schedule:
+    - cron: '0 12 * * 0'
+
+jobs:
+  gmt_cache:
+    name: Cache GMT artifacts
+    runs-on: macOS-latest
+
+    steps:
+      # Setup Miniconda
+      - name: Setup Miniconda
+        uses: goanpeca/[email protected]
+        with:
+          channels: conda-forge
+
+      # Install GMT
+      - name: Install GMT
+        shell: bash -l {0}
+        run: conda install -c conda-forge gmt=6.1.0
+
+      # Download remote files
+      - name: Download remote data
+        shell: bash -l {0}
+        run: |
+          gmt which -Ga @earth_relief_10m_p @earth_relief_10m_g @earth_relief_30m_p @earth_relief_30m_g @earth_relief_01d_p @earth_relief_01d_g
+          gmt which -Ga @ridge.txt @Table_5_11.txt @test.dat.nc @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt
+
+      # Upload the downloaded files as artifacts to Github
+      - name: Upload artifacts to Github
+        uses: actions/upload-artifact@v2
+        with:
+          name: gmt-cache
+          path: |
+              ~/.gmt/cache
+              ~/.gmt/server

.github/workflows/ci_tests.yaml

@@ -83,39 +83,21 @@ jobs:
         shell: bash -l {0}
         run: conda list
 
-      # Cache the ${HOME}/.gmt directory, for docs and testing
-      - name: Cache GMT directory
-        uses: actions/cache@v2
-        id: cache
+      # Download cached remote files (artifacts) from Github
+      - name: Download remote data from Github
+        uses: dawidd6/[email protected]
         with:
-          path: |
-            ~/.gmt/cache
-            ~/.gmt/server
-          key: cache-gmt-${{ runner.os }}-${{ github.ref }}-20200711
-          restore-keys: cache-gmt-${{ runner.os }}-refs/heads/master-
-
-      # Workaround for the timeouts of 'gmt which' on Linux and Windows
-      - name: Download remote data using wget (Linux & Windows)
-        shell: bash -l {0}
-        run: |
-          if [ "$RUNNER_OS" == "Windows" ]; then choco install wget; fi  # install wget on Windows
-          mkdir -p ~/.gmt ~/.gmt/cache ~/.gmt/server/earth/earth_relief/
-          wget --no-check-certificate https://oceania.generic-mapping-tools.org/gmt_hash_server.txt -P ~/.gmt/server/
-          for data in earth_relief_01d_p.grd earth_relief_01d_g.grd earth_relief_30m_p.grd earth_relief_30m_g.grd earth_relief_10m_p.grd earth_relief_10m_g.grd; do
-            wget --no-check-certificate https://oceania.generic-mapping-tools.org/server/earth/earth_relief/${data} -P ~/.gmt/server/earth/earth_relief/
-          done
-          for data in ridge.txt Table_5_11.txt test.dat.nc tut_bathy.nc tut_quakes.ngdc tut_ship.xyz usgs_quakes_22.txt; do
-            wget --no-check-certificate https://oceania.generic-mapping-tools.org/cache/${data} -P ~/.gmt/cache/
-          done
-        if: steps.cache.outputs.cache-hit != 'true' && runner.os != 'macOS'
-
-      # Download remote files, if not already cached
-      - name: Download remote data (macOS)
+          workflow: cache_data.yaml
+          name: gmt-cache
+          path: .gmt
+
+      # Move downloaded files to ~/.gmt directory and list them
+      - name: Move and list downloaded remote files
         shell: bash -l {0}
         run: |
-          gmt which -Ga @earth_relief_10m_p @earth_relief_10m_g @earth_relief_30m_p @earth_relief_30m_g @earth_relief_01d_p @earth_relief_01d_g
-          gmt which -Ga @ridge.txt @Table_5_11.txt @test.dat.nc @tut_bathy.nc @tut_quakes.ngdc @tut_ship.xyz @usgs_quakes_22.txt
-        if: steps.cache.outputs.cache-hit != 'true' && runner.os == 'macOS'
+          mkdir -p ~/.gmt
+          mv .gmt/* ~/.gmt
+          ls -lh ~/.gmt
 
       # Install the package that we want to test
       - name: Install the package

MAINTENANCE.md

@@ -46,6 +46,10 @@ The configuration file for these services are in
 `.github/workflows/ci_tests.yaml` and `.travis.yml`.
 They rely on the `requirements.txt` file to install required dependencies using
 conda and the `Makefile` to run the tests and checks.
+There is also a `.github/workflows/cache_data.yaml` file which caches some GMT
+remote data files needed for the Github Actions CI on a regular basis.
+If new remote files are needed urgently, maintainers can manually uncomment
+the 'pull_request:' line in that `cache_data.yaml` file to refresh the cache.
 
 Travis also handles all of our deployments automatically: