Merge pull request #120 from CIROH-UA/test_terraform

test terraform

.github/workflows/datastream_python.yml

@@ -4,18 +4,22 @@
 name: Test Datastream Python
 on:
   push:
-    branches: [ "main" ]
+    branches:
+      - main
+    paths:
+      - 'python_tools/**'
+      - '.github/workflows/datastream_python.yml'
   pull_request:
-    branches: [ "main" ]
-
+    branches:
+      - main    
+    paths:      
+      - 'python_tools/**'
+      - '.github/workflows/datastream_python.yml'
 permissions:
   contents: read
-
 jobs:
   build:
-
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.10

.github/workflows/forcingprocessor.yml

@@ -4,18 +4,24 @@
 name: Test Forcing Processor
 on:
   push:
-    branches: [ "main" ]
+    branches:
+      - main
+    paths:
+      - 'forcingprocessor/**'
+      - '.github/workflows/forcingprocessor.yml'
   pull_request:
-    branches: [ "main" ]
-
+    branches:
+      - main    
+    paths:      
+      - 'forcingprocessor/**'
+      - '.github/workflows/forcingprocessor.yml'
 permissions:
   contents: read
-
 jobs:
   build:
 
     runs-on: ubuntu-latest
-
+    timeout-minutes: 10
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3.10

.github/workflows/research_datastream_terraform.yml

@@ -0,0 +1,34 @@
+name: Validate research datastream terraform
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'research_datastream/terraform/**'
+      - '.github/workflows/research_datastream_terraform.yml'
+  pull_request:
+    branches:
+      - main    
+    paths:      
+      - 'research_datastream/terraform/**'
+      - '.github/workflows/research_datastream_terraform.yml'
+jobs:
+  validate-terraform:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v1
+      - name: Configure AWS
+        run: |
+          aws configure set aws_access_key_id ${{ secrets.aws_access_key_id }}
+          aws configure set aws_secret_access_key ${{ secrets.aws_secret_access_key }}
+          aws configure set region us-east-1        
+      - name: Validate Terraform
+        run: |
+          cd research_datastream/terraform
+          terraform init
+          terraform validate
+          terraform apply -var-file=./variables_gitactions.tfvars -auto-approve
+          terraform destroy -var-file=./variables_gitactions.tfvars -auto-approve

forcingprocessor/tests/test_forcingprocessor.py

@@ -117,7 +117,6 @@ def test_gs():
     os.remove(parquet)    
 
 def test_gcs():
-    # assert False, f'hangs in pytest, but should work'
     nwmurl_conf['start_date'] = "202407100100"
     nwmurl_conf['end_date']   = "202407100100" 
     nwmurl_conf["urlbaseinput"] = 6
@@ -127,7 +126,7 @@ def test_gcs():
     assert parquet.exists()
     os.remove(parquet)        
 
-def test_noaa_nwm_pds():
+def test_noaa_nwm_pds_https():
     nwmurl_conf['start_date'] = date + hourminute
     nwmurl_conf['end_date']   = date + hourminute    
     nwmurl_conf["urlbaseinput"] = 7
@@ -158,7 +157,7 @@ def test_ciroh_zarr():
     assert parquet.exists()
     os.remove(parquet)     
 
-def test_retro_2_1():
+def test_retro_2_1_https():
     conf['forcing']['nwm_file'] = retro_filenamelist
     nwmurl_conf_retro["urlbaseinput"] = 1
     generate_nwmfiles(nwmurl_conf_retro)

research_datastream/terraform/variables.tfvars

@@ -1,4 +1,4 @@
-region                = "us-west-2"
+region                = "us-east-1"
 sm_name               = "datastreamconductor_test"
 sm_role_name          = "datastreamconductor_role_test"
 starter_lambda_name   = "ec2start_ami_test"

research_datastream/terraform/variables_gitactions.tfvars

@@ -0,0 +1,14 @@
+region                = "us-east-1"
+sm_name               = "datastreamconductor_github_actions"
+sm_role_name          = "datastreamconductor_role_github_actions"
+starter_lambda_name   = "ec2start_ami_github_actions"
+commander_lambda_name = "ec2commander_github_actions"
+poller_lambda_name    = "CommandPoller_github_actions"
+checker_lambda_name   = "S3ObjectChecker_github_actions"
+stopper_lambda_name   = "EC2Stopper_github_actions"
+lambda_policy_name    = "datastream_lambda_policy_github_actions"
+lambda_role_name      = "datastream_lambda_role_github_actions"
+lambda_invoke_policy_name = "datastream_lambda_invoke_policy_github_actions"
+ec2_role              = "datastream_ec2_role_github_actions"
+ec2_policy_name       = "datastream_ec2_policy_github_actions"
+profile_name          = "datastream_ec2_profile_github_actions"