Added pytest for functional tests, reading executor config from env (#56

)

* Added pytest for functional tests, reading executor config from env

* marking functional tests with functional_tests

* added pytest markers to pyproject.toml

* added svm workflow to functional tests

* updated functional test executor import

* Updated REAMDE

* Updated ft README

.env.example

@@ -0,0 +1,5 @@
+executor_username=
+executor_conda_env=
+# terraform outputs
+executor_hostname=
+executor_ssh_key_file=

.github/workflows/tests.yml

@@ -89,7 +89,7 @@ jobs:
 
       - name: Run tests
         run: |
-          PYTHONPATH=$PWD/tests pytest -vv tests/ --cov=covalent_ssh_plugin
+          PYTHONPATH=$PWD/tests pytest -m "not functional_tests" -vv tests/  --cov=covalent_ssh_plugin
 
       - name: Generate coverage report
         run: coverage xml -o coverage.xml

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [UNRELEASED]
 
+### Changed
+
+- Functional tests using pytest and .env file configuration
+
 ## [0.17.0] - 2022-10-28
 
 ### Changed

pyproject.toml

@@ -37,3 +37,8 @@ multi_line_output = 3
 include_trailing_comma = true
 profile = 'black'
 skip_gitignore = true
+
+[tool.pytest.ini_options]
+markers = [
+    "functional_tests: marks tests that are to be run in the functional tests ci pipeline"
+]

tests/functional_tests/README.md

@@ -0,0 +1,25 @@
+## Functional Test Instructions
+
+### 1.Setup
+
+In the project root run the following:
+
+```sh
+pip install -r ./tests/requirements.txt
+pip install -r ./tests/functional_tests/requirements.txt
+export PYTHONPATH=$(pwd)
+```
+
+Copy create `.env` file:
+
+```sh
+cp .env.example .env
+```
+
+Fill in the configuration values either manually or from terraform output.
+
+### 2. Run Functional Tests
+
+```sh
+pytest -vvs -m functional_tests
+```

tests/functional_tests/basic_workflow_test.py

@@ -0,0 +1,31 @@
+import covalent as ct
+import pytest
+
+from tests.functional_tests.fixtures.executor import executor
+
+# Basic Workflow
+
+
+@pytest.mark.functional_tests
+def test_basic_workflow():
+    @ct.electron(executor=executor)
+    def join_words(a, b):
+        return ", ".join([a, b])
+
+    @ct.electron
+    def excitement(a):
+        return f"{a}!"
+
+    @ct.lattice
+    def basic_workflow(a, b):
+        phrase = join_words(a, b)
+        return excitement(phrase)
+
+    # Dispatch the workflow
+    dispatch_id = ct.dispatch(basic_workflow)("Hello", "World")
+    result = ct.get_result(dispatch_id=dispatch_id, wait=True)
+    status = str(result.status)
+
+    print(result)
+
+    assert status == str(ct.status.COMPLETED)

tests/functional_tests/fixtures/executor.py

@@ -0,0 +1,19 @@
+from dotenv import load_dotenv
+
+load_dotenv()
+
+import os
+
+from covalent_ssh_plugin import SSHExecutor
+
+executor_config = {
+    "username": os.getenv("executor_username", "ubuntu"),
+    "hostname": os.getenv("executor_hostname"),
+    "ssh_key_file": os.getenv("executor_ssh_key_file", ""),
+    "conda_env": os.getenv("executor_conda_env", "covalent"),
+}
+
+print("Using Executor Configuration:")
+print(executor_config)
+
+executor = SSHExecutor(**executor_config)

tests/functional_tests/requirements.txt

@@ -1,2 +1,3 @@
 numpy==1.23.2
+python-dotenv==0.21.0
 scikit-learn==1.1.2

tests/functional_tests/svm_workflow.py

@@ -1,52 +1,48 @@
-import sys
-
 import covalent as ct
-import executor_instance
+import pytest
 from numpy.random import permutation
 from sklearn import datasets, svm
 
-deps_pip = ct.DepsPip(packages=["numpy==1.23.2", "scikit-learn==1.1.2"])
-
-
-@ct.electron
-def load_data():
-    iris = datasets.load_iris()
-    perm = permutation(iris.target.size)
-    iris.data = iris.data[perm]
-    iris.target = iris.target[perm]
-    return iris.data, iris.target
-
-
-@ct.electron(executor=executor_instance.executor, deps_pip=deps_pip)
-def train_svm(data, C, gamma):
-    X, y = data
-    clf = svm.SVC(C=C, gamma=gamma)
-    clf.fit(X[90:], y[90:])
-    return clf
-
+from tests.functional_tests.fixtures.executor import executor
 
-@ct.electron
-def score_svm(data, clf):
-    X_test, y_test = data
-    return clf.score(X_test[:90], y_test[:90])
-
-
-@ct.lattice
-def run_experiment(C=1.0, gamma=0.7):
-    data = load_data()
-    clf = train_svm(data=data, C=C, gamma=gamma)
-    score = score_svm(data=data, clf=clf)
-    return score
-
-
-dispatchable_func = ct.dispatch(run_experiment)
-
-dispatch_id = dispatchable_func(C=1.0, gamma=0.7)
-result = ct.get_result(dispatch_id=dispatch_id, wait=True)
-status = str(result.status)
+deps_pip = ct.DepsPip(packages=["numpy==1.23.2", "scikit-learn==1.1.2"])
 
-print(result)
 
-if status == str(ct.status.FAILED):
-    print("Basic Workflow failed to run.")
-    sys.exit(1)
+@pytest.mark.functional_tests
+def test_svm_workflow():
+    @ct.electron
+    def load_data():
+        iris = datasets.load_iris()
+        perm = permutation(iris.target.size)
+        iris.data = iris.data[perm]
+        iris.target = iris.target[perm]
+        return iris.data, iris.target
+
+    @ct.electron(executor=executor, deps_pip=deps_pip)
+    def train_svm(data, C, gamma):
+        X, y = data
+        clf = svm.SVC(C=C, gamma=gamma)
+        clf.fit(X[90:], y[90:])
+        return clf
+
+    @ct.electron
+    def score_svm(data, clf):
+        X_test, y_test = data
+        return clf.score(X_test[:90], y_test[:90])
+
+    @ct.lattice
+    def run_experiment(C=1.0, gamma=0.7):
+        data = load_data()
+        clf = train_svm(data=data, C=C, gamma=gamma)
+        score = score_svm(data=data, clf=clf)
+        return score
+
+    dispatchable_func = ct.dispatch(run_experiment)
+
+    dispatch_id = dispatchable_func(C=1.0, gamma=0.7)
+    result = ct.get_result(dispatch_id=dispatch_id, wait=True)
+    status = str(result.status)
+
+    print(result)
+
+    assert status == str(ct.status.COMPLETED)