Workflow GitHub actions (#70)

* github actions linting for workflows

* trigger on push

* try on pull requests

* test change workflows directory

* fix test call to exclude SWIPE tests

* replace model exports

* add AWS_REGION

* fix lint errors

.github/workflows/workflows-github-tests.yml

@@ -0,0 +1,42 @@
+name: Workflows tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - "workflows/**"
+  pull_request:
+    branches: "**"
+    paths:
+      - "workflows/**"
+
+jobs:
+  py-lint:
+    runs-on: [ARM64, self-hosted, Linux]
+    steps:
+      - uses: actions/checkout@v3
+      # Build images and run linters
+      - name: Set up deps
+        run: |
+          make gha-setup
+      - name: Lint Python
+        working-directory: workflows
+        run: |
+          make local-build
+          make check-lint
+  py-test:
+    runs-on: [ARM64, self-hosted, Linux]
+    steps:
+      - uses: actions/checkout@v3
+      # Set up a local dev env and run tests
+      - name: Set up deps
+        run: |
+          make gha-setup
+      - name: Run tests
+        working-directory: workflows
+        run: |
+          make local-init
+          make local-db-tests
+        env: 
+          AWS_REGION: us-west-2

workflows/Dockerfile

@@ -18,7 +18,7 @@ ENV PYTHONPATH=.
 COPY workflows/pyproject.toml workflows/poetry.lock ./
 RUN poetry install
 
-# Ordering is important here. Entities first!
+# Ordering is important here. Workflows first!
 COPY workflows/ .
 COPY platformics ./platformics
 

workflows/Makefile

@@ -53,6 +53,11 @@ local-start: local-envfile ## Start a local dev environment that's been stopped.
 local-pgconsole: ## Connect to the local postgres database.
 	$(docker_compose) exec postgres psql "$(LOCAL_DB_CONN_STRING)"
 
+
+.PHONY: local-db-tests
+local-db-tests: ## Run workflow db tests
+	$(docker_compose_run) $(FOLDER) bash -c "poetry run pytest test/"
+
 ### SWIPE Plugin Tests
 
 .PHONY: local-swipe-plugin-tests

workflows/database/models/__init__.py

@@ -1,4 +1,11 @@
 from platformics.database.models.base import Base, meta
-from database.models.workflow import Workflow, WorkflowVersion, Run, RunStatus, RunStep, RunEntityInput
+from database.models.workflow import (
+    Workflow,
+    WorkflowVersion,
+    RunStatus,
+    Run,
+    RunStep,
+    RunEntityInput,
+)
 
 __all__ = ["Base", "meta", "Workflow", "WorkflowVersion", "Run", "RunStatus", "RunStep", "RunEntityInput"]

workflows/terraform_test/run_swipe_plugin_test.py

@@ -2,16 +2,16 @@
 import time
 import json
 import boto3
-from typing import Dict, List
+from typing import Dict, List, Any
 from workflow_runner_swipe import SwipeWorkflowRunner
 
 
 class AWSMock:
     def __init__(
         self,
-        endpoint_url="http://motoserver.czidnet:4000",
-        sfn_endpoint_url="http://sfn.czidnet:8083",
-        aws_region="us-east-1",
+        endpoint_url: str = "http://motoserver.czidnet:4000",
+        sfn_endpoint_url: str = "http://sfn.czidnet:8083",
+        aws_region: str = "us-east-1",
     ) -> None:
         self.s3 = boto3.resource("s3", endpoint_url=endpoint_url, region_name=aws_region)
         self.sqs = boto3.client("sqs", endpoint_url=endpoint_url, region_name=aws_region)
@@ -38,7 +38,7 @@ def retrieve_message(self, url: str) -> Dict:
         )
         return json.loads(message["Body"])
 
-    def get_sfn_execution_status(self, sfn_arn) -> List:
+    def get_sfn_execution_status(self, sfn_arn: str) -> List:
         return self.sfn.describe_execution(executionArn=sfn_arn)["status"]
 
 
@@ -60,7 +60,7 @@ def setUp(self) -> None:
         self.input_obj.put(Body="hello".encode())
         self.aws = AWSMock()
 
-    def print_execution(self, events):
+    def print_execution(self, events: List[Any]) -> None:
         import sys
 
         seen_events = set()