Fetch pathway figures and metadata #920
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fetch pathway figures and metadata | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "30 */6 * 1,4,7,10 *" #Runs every 6 hours at 30 past the hour, everyday of Jan, Apr, Jul, Oct. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: false # to allow multiple runs to queue up rather than clobber | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| status: ${{ steps.early.outputs.status }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - id: early | |
| name: Date check | |
| run: | | |
| last_run_date=$(sed -n 's/^last_run:[[:space:]]*\([[:digit:]]\{4\}\/[[:digit:]]\{2\}\/[[:digit:]]\{2\}\).*$/\1/p' query_config.yml) | |
| today_date="$(date +'%Y/%m/%d')" | |
| if [[ "$last_run_date" < "$today_date" ]]; then | |
| echo "status=proceed" >> $GITHUB_OUTPUT | |
| else | |
| echo "status=quit" >> $GITHUB_OUTPUT | |
| echo "Nothing more to collect at this time (see last_run date). Skipping other steps..." | |
| fi | |
| scrape: | |
| needs: check | |
| if: ${{needs.check.outputs.status == 'proceed' }} | |
| runs-on: ubuntu-latest | |
| services: | |
| selenium: | |
| image: selenium/standalone-firefox:4.8.3-20230328 | |
| ports: | |
| - 4445:4444 | |
| options: --shm-size="2g" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Setup R | |
| uses: r-lib/actions/setup-r@v2 | |
| - name: Install Packages | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libcurl4-openssl-dev | |
| Rscript -e 'install.packages(c("RSelenium","utils","httr","xml2","dplyr","magrittr","stringr","purrr","yaml","rvest","lubridate","jpeg"))' | |
| - name: Run R Script to Fetch Figures | |
| run: Rscript scripts/fetch_figures.R || true #commit files even if R crashes | |
| - name: Commit, pull with rebase (any remote changes during this run), and then push | |
| run: | | |
| git config --global user.name 'GitHub Action' | |
| git config --global user.email '[email protected]' | |
| git add . | |
| if git diff --exit-code --staged; then | |
| echo "No changes" | |
| else | |
| git commit -m 'new figures from pmc; updated config and log' | |
| git pull --rebase | |
| git push --force-with-lease | |
| fi | |