test_datastream_options.yml

name: Test Datastream Options

on:
  push:
    branches:
      - main

  pull_request:
    branches:
        - main  

jobs:
  test-datastream-options:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v2  

    - name: Set up QEMU
      uses: docker/setup-qemu-action@v3

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3      

    - name: Configure AWS
      run: |
        aws configure set aws_access_key_id ${{ secrets.aws_access_key_id }}
        aws configure set aws_secret_access_key ${{ secrets.aws_secret_access_key }}
        aws configure set region us-east-1    

    - name: Build docker containers
      run : |        
        ./scripts/docker_builds.sh -b        
        
    - name: Install packages for datastream
      run: |
        curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset && sudo apt-get install git pip pigz awscli python3.9 -y        

    - name: Get geopackage from hfsubset
      run: |
        hfsubset -w medium_range -s nextgen -v 2.1.1 -l divides,flowlines,network,nexus,forcing-weights,flowpath-attributes,model-attributes -o palisade.gpkg -t hl "Gages-09106150"

    - name: Base test and NWM_RETRO_V3
      run: |        
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json

    - name: Cache resource directory
      run: |
        mkdir ./data/cache
        cp -r ./data/datastream_test/datastream-resources ./data/cache
        cp -r ./data/cache/datastream-resources ./data/cache/datastream-resources-no-forcings
        cp -r ./data/cache/datastream-resources ./data/cache/datastream-resources-missing
        sudo rm -rf ./data/cache/datastream-resources-no-forcings/ngen-forcings 

    - name: Resource directory test missing all
      if: always()
      run: |
        sudo rm -rf $(pwd)/data/datastream_test
        sudo rm -rf ./data/cache/datastream-resources-missing/ngen-forcings 
        sudo rm -rf ./data/cache/datastream-resources-missing/config/*
        ./scripts/stream.sh -r ./data/cache/datastream-resources-missing -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg

    - name: Resource directory test
      if: always()
      run: |
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -r ./data/cache/datastream-resources -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test

    - name: Forcings sources option test NWM_RETRO_V2
      if: always()
      run: |
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -r ./data/cache/datastream-resources-no-forcings -s 201906200100 -e 201906200200 -C NWM_RETRO_V2 -d $(pwd)/data/datastream_test        
        
    
    - name: Forcings sources option test NWM_OPERATIONAL_V3
      if: always()
      run: |
        sudo rm -rf $(pwd)/data/datastream_test
        TODAY=$(env TZ=US/Eastern date +'%Y%m%d')
        ./scripts/stream.sh -r ./data/cache/datastream-resources-no-forcings -s $TODAY"0100" -e $TODAY"0200" -C NWM_OPERATIONAL_V3 -d $(pwd)/data/datastream_test   
        
    - name: Forcings sources option test NOMADS_OPERATIONAL
      if: always()
      run: |        
        sudo rm -rf $(pwd)/data/datastream_test
        TODAY=$(env TZ=US/Eastern date +'%Y%m%d')
        ./scripts/stream.sh -r ./data/cache/datastream-resources-no-forcings -s $TODAY"0100" -e $TODAY"0200" -C NOMADS_OPERATIONAL -d $(pwd)/data/datastream_test  
        
    - name: Test hfsubset options
      if: always()
      run: |
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -I "Gages-09106150" -i hl -v 2.1.1 -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json

    - name: S3 write out test
      if: always()
      run: |        
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json -S ngen-datastream -o git_actions_test/
        aws s3api wait object-exists --bucket ngen-datastream --key git_actions_test/ngen-run.tar.gz   
        aws s3api delete-object --bucket ngen-datastream --key git_actions_test/ngen-run.tar.gz

    - name: DAILY today test
      if: always()
      run: |        
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -s DAILY -C NWM_OPERATIONAL_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json

    - name: DAILY pick day test
      if: always()
      run: |        
        sudo rm -rf $(pwd)/data/datastream_test
        ./scripts/stream.sh -s DAILY -e 202006200000 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json