From 5ac7b9ca2b789d83b6892f7274501c6823b9b010 Mon Sep 17 00:00:00 2001 From: Alejandro Esquivel Date: Mon, 5 Dec 2022 12:48:16 -0800 Subject: [PATCH] Added pytest for functional tests, reading executor config from env (#56) * Added pytest for functional tests, reading executor config from env * marking functional tests with functional_tests * added pytest markers to pyproject.toml * added svm workflow to functional tests * updated functional test executor import * Updated REAMDE * Updated ft README --- .env.example | 5 ++ .github/workflows/tests.yml | 2 +- CHANGELOG.md | 4 + pyproject.toml | 5 ++ tests/functional_tests/README.md | 25 ++++++ tests/functional_tests/basic_workflow.py | 36 -------- tests/functional_tests/basic_workflow_test.py | 31 +++++++ tests/functional_tests/executor_instance.py | 21 ----- tests/functional_tests/fixtures/__init__.py | 0 tests/functional_tests/fixtures/executor.py | 19 ++++ tests/functional_tests/requirements.txt | 1 + tests/functional_tests/svm_workflow.py | 86 +++++++++---------- tests/functional_tests/terraform_output.py | 33 ------- 13 files changed, 132 insertions(+), 136 deletions(-) create mode 100644 .env.example create mode 100644 tests/functional_tests/README.md delete mode 100644 tests/functional_tests/basic_workflow.py create mode 100644 tests/functional_tests/basic_workflow_test.py delete mode 100644 tests/functional_tests/executor_instance.py create mode 100644 tests/functional_tests/fixtures/__init__.py create mode 100644 tests/functional_tests/fixtures/executor.py delete mode 100644 tests/functional_tests/terraform_output.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a54eef0 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +executor_username= +executor_conda_env= +# terraform outputs +executor_hostname= +executor_ssh_key_file= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 97ca810..f6e1cdc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -89,7 +89,7 @@ jobs: - name: Run tests run: | - PYTHONPATH=$PWD/tests pytest -vv tests/ --cov=covalent_ssh_plugin + PYTHONPATH=$PWD/tests pytest -m "not functional_tests" -vv tests/ --cov=covalent_ssh_plugin - name: Generate coverage report run: coverage xml -o coverage.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index cf06583..2b87cba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [UNRELEASED] +### Changed + +- Functional tests using pytest and .env file configuration + ## [0.17.0] - 2022-10-28 ### Changed diff --git a/pyproject.toml b/pyproject.toml index 0c31a1e..a23f117 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,3 +37,8 @@ multi_line_output = 3 include_trailing_comma = true profile = 'black' skip_gitignore = true + +[tool.pytest.ini_options] +markers = [ + "functional_tests: marks tests that are to be run in the functional tests ci pipeline" +] diff --git a/tests/functional_tests/README.md b/tests/functional_tests/README.md new file mode 100644 index 0000000..05c15f7 --- /dev/null +++ b/tests/functional_tests/README.md @@ -0,0 +1,25 @@ +## Functional Test Instructions + +### 1.Setup + +In the project root run the following: + +```sh +pip install -r ./tests/requirements.txt +pip install -r ./tests/functional_tests/requirements.txt +export PYTHONPATH=$(pwd) +``` + +Copy create `.env` file: + +```sh +cp .env.example .env +``` + +Fill in the configuration values either manually or from terraform output. + +### 2. Run Functional Tests + +```sh +pytest -vvs -m functional_tests +``` diff --git a/tests/functional_tests/basic_workflow.py b/tests/functional_tests/basic_workflow.py deleted file mode 100644 index 6a7c107..0000000 --- a/tests/functional_tests/basic_workflow.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys - -import covalent as ct - -# Extract terraform outputs & instantiate executor -import executor_instance - -# Basic Workflow - - -@ct.electron(executor=executor_instance.executor) -def join_words(a, b): - return ", ".join([a, b]) - - -@ct.electron -def excitement(a): - return f"{a}!" - - -@ct.lattice -def basic_workflow(a, b): - phrase = join_words(a, b) - return excitement(phrase) - - -# Dispatch the workflow -dispatch_id = ct.dispatch(basic_workflow)("Hello", "World") -result = ct.get_result(dispatch_id=dispatch_id, wait=True) -status = str(result.status) - -print(result) - -if status == str(ct.status.FAILED): - print("Basic Workflow failed to run.") - sys.exit(1) diff --git a/tests/functional_tests/basic_workflow_test.py b/tests/functional_tests/basic_workflow_test.py new file mode 100644 index 0000000..d681580 --- /dev/null +++ b/tests/functional_tests/basic_workflow_test.py @@ -0,0 +1,31 @@ +import covalent as ct +import pytest + +from tests.functional_tests.fixtures.executor import executor + +# Basic Workflow + + +@pytest.mark.functional_tests +def test_basic_workflow(): + @ct.electron(executor=executor) + def join_words(a, b): + return ", ".join([a, b]) + + @ct.electron + def excitement(a): + return f"{a}!" + + @ct.lattice + def basic_workflow(a, b): + phrase = join_words(a, b) + return excitement(phrase) + + # Dispatch the workflow + dispatch_id = ct.dispatch(basic_workflow)("Hello", "World") + result = ct.get_result(dispatch_id=dispatch_id, wait=True) + status = str(result.status) + + print(result) + + assert status == str(ct.status.COMPLETED) diff --git a/tests/functional_tests/executor_instance.py b/tests/functional_tests/executor_instance.py deleted file mode 100644 index 920afaf..0000000 --- a/tests/functional_tests/executor_instance.py +++ /dev/null @@ -1,21 +0,0 @@ -import os - -import covalent as ct -import terraform_output - -hostname = os.getenv("SSH_EXECUTOR_HOSTNAME") or terraform_output.get("ec2_public_ip", "") -username = os.getenv("SSH_EXECUTOR_USERNAME", "ubuntu") -ssh_key_file = os.getenv("SSH_EXECUTOR_SSH_KEY_FILE", "") -conda_env = os.getenv("SSH_EXECUTOR_CONDA_ENV", "covalent") - -executor_config = { - "username": username, - "hostname": hostname, - "ssh_key_file": ssh_key_file, - "conda_env": conda_env, -} - -print("Using Executor Config:") -print(executor_config) - -executor = ct.executor.SSHExecutor(**executor_config) diff --git a/tests/functional_tests/fixtures/__init__.py b/tests/functional_tests/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/functional_tests/fixtures/executor.py b/tests/functional_tests/fixtures/executor.py new file mode 100644 index 0000000..2cfbcb2 --- /dev/null +++ b/tests/functional_tests/fixtures/executor.py @@ -0,0 +1,19 @@ +from dotenv import load_dotenv + +load_dotenv() + +import os + +from covalent_ssh_plugin import SSHExecutor + +executor_config = { + "username": os.getenv("executor_username", "ubuntu"), + "hostname": os.getenv("executor_hostname"), + "ssh_key_file": os.getenv("executor_ssh_key_file", ""), + "conda_env": os.getenv("executor_conda_env", "covalent"), +} + +print("Using Executor Configuration:") +print(executor_config) + +executor = SSHExecutor(**executor_config) diff --git a/tests/functional_tests/requirements.txt b/tests/functional_tests/requirements.txt index 66a7502..ccb9b2b 100644 --- a/tests/functional_tests/requirements.txt +++ b/tests/functional_tests/requirements.txt @@ -1,2 +1,3 @@ numpy==1.23.2 +python-dotenv==0.21.0 scikit-learn==1.1.2 diff --git a/tests/functional_tests/svm_workflow.py b/tests/functional_tests/svm_workflow.py index b97c91c..030ed97 100644 --- a/tests/functional_tests/svm_workflow.py +++ b/tests/functional_tests/svm_workflow.py @@ -1,52 +1,48 @@ -import sys - import covalent as ct -import executor_instance +import pytest from numpy.random import permutation from sklearn import datasets, svm -deps_pip = ct.DepsPip(packages=["numpy==1.23.2", "scikit-learn==1.1.2"]) - - -@ct.electron -def load_data(): - iris = datasets.load_iris() - perm = permutation(iris.target.size) - iris.data = iris.data[perm] - iris.target = iris.target[perm] - return iris.data, iris.target - - -@ct.electron(executor=executor_instance.executor, deps_pip=deps_pip) -def train_svm(data, C, gamma): - X, y = data - clf = svm.SVC(C=C, gamma=gamma) - clf.fit(X[90:], y[90:]) - return clf - +from tests.functional_tests.fixtures.executor import executor -@ct.electron -def score_svm(data, clf): - X_test, y_test = data - return clf.score(X_test[:90], y_test[:90]) - - -@ct.lattice -def run_experiment(C=1.0, gamma=0.7): - data = load_data() - clf = train_svm(data=data, C=C, gamma=gamma) - score = score_svm(data=data, clf=clf) - return score - - -dispatchable_func = ct.dispatch(run_experiment) - -dispatch_id = dispatchable_func(C=1.0, gamma=0.7) -result = ct.get_result(dispatch_id=dispatch_id, wait=True) -status = str(result.status) +deps_pip = ct.DepsPip(packages=["numpy==1.23.2", "scikit-learn==1.1.2"]) -print(result) -if status == str(ct.status.FAILED): - print("Basic Workflow failed to run.") - sys.exit(1) +@pytest.mark.functional_tests +def test_svm_workflow(): + @ct.electron + def load_data(): + iris = datasets.load_iris() + perm = permutation(iris.target.size) + iris.data = iris.data[perm] + iris.target = iris.target[perm] + return iris.data, iris.target + + @ct.electron(executor=executor, deps_pip=deps_pip) + def train_svm(data, C, gamma): + X, y = data + clf = svm.SVC(C=C, gamma=gamma) + clf.fit(X[90:], y[90:]) + return clf + + @ct.electron + def score_svm(data, clf): + X_test, y_test = data + return clf.score(X_test[:90], y_test[:90]) + + @ct.lattice + def run_experiment(C=1.0, gamma=0.7): + data = load_data() + clf = train_svm(data=data, C=C, gamma=gamma) + score = score_svm(data=data, clf=clf) + return score + + dispatchable_func = ct.dispatch(run_experiment) + + dispatch_id = dispatchable_func(C=1.0, gamma=0.7) + result = ct.get_result(dispatch_id=dispatch_id, wait=True) + status = str(result.status) + + print(result) + + assert status == str(ct.status.COMPLETED) diff --git a/tests/functional_tests/terraform_output.py b/tests/functional_tests/terraform_output.py deleted file mode 100644 index 7ecf7e8..0000000 --- a/tests/functional_tests/terraform_output.py +++ /dev/null @@ -1,33 +0,0 @@ -import json -import os -import subprocess -import sys -import traceback - -TERRAFORM_OUTPUTS = {} - -try: - terraform_dir = os.getenv("TF_DIR") - proc = subprocess.run( - [ - "terraform", - f"-chdir={terraform_dir}", - "output", - "-json", - ], - check=True, - capture_output=True, - ) - TERRAFORM_OUTPUTS = json.loads(proc.stdout.decode()) -except Exception: - traceback.print_exc() - -print("Terraform output:") -print(TERRAFORM_OUTPUTS) - - -def get(key: str, default): - try: - return TERRAFORM_OUTPUTS[key]["value"] - except KeyError: - return default