From a3f9a8b600fba6b7ddacc0457bdfdf92acb98547 Mon Sep 17 00:00:00 2001 From: Renfu Li Date: Tue, 19 Dec 2023 19:36:13 -0800 Subject: [PATCH 1/3] fix: make outPath accept both str and Path --- src/onc/modules/_DataProductFile.py | 3 +- src/onc/modules/_OncArchive.py | 7 ++-- src/onc/modules/_util.py | 18 ++++------ src/onc/onc.py | 19 +++++------ tests/README.md | 50 ++++++++++++++++------------ tests/robot/libraries/delivery.py | 4 ++- tests/robot/suites/00__initial.robot | 2 +- 7 files changed, 54 insertions(+), 49 deletions(-) diff --git a/src/onc/modules/_DataProductFile.py b/src/onc/modules/_DataProductFile.py index 6171fcd..a653991 100644 --- a/src/onc/modules/_DataProductFile.py +++ b/src/onc/modules/_DataProductFile.py @@ -1,3 +1,4 @@ +from pathlib import Path from time import sleep, time from warnings import warn @@ -41,7 +42,7 @@ def download( self, timeout: int, pollPeriod: float, - outPath: str, + outPath: Path, maxRetries: int, overwrite: bool, ): diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index d65d7dd..7a3b5de 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -1,5 +1,6 @@ import os import time +from pathlib import Path import humanize import requests @@ -50,7 +51,7 @@ def getFile(self, filename: str = "", overwrite: bool = False): if response.ok: # Save file to output path - outPath = self._config("outPath") + outPath: Path = self._config("outPath") saveAsFile(response, outPath, filename, overwrite) else: @@ -109,8 +110,8 @@ def getDirectFiles( downInfos = [] for filename in dataRows["files"]: # only download if file doesn't exist (or overwrite is True) - outPath = self._config("outPath") - filePath = f"{outPath}/{filename}" + outPath: Path = self._config("outPath") + filePath = outPath / filename fileExists = os.path.exists(filePath) if (not fileExists) or (fileExists and overwrite): diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index c5ca810..6b847d4 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -1,28 +1,24 @@ -import os from datetime import timedelta +from pathlib import Path import humanize import requests def saveAsFile( - response: requests.Response, filePath: str, fileName: str, overwrite: bool + response: requests.Response, outPath: Path, fileName: str, overwrite: bool ) -> None: """ Saves the file downloaded in the response object, in the outPath, with filename If overwrite, will overwrite files with the same name """ - fullPath = fileName - if len(filePath) > 0: - fullPath = filePath + "/" + fileName - # Create outPath directory if not exists - if not os.path.exists(filePath): - os.makedirs(filePath) + filePath = outPath / fileName + outPath.mkdir(parents=True, exist_ok=True) # Save file in outPath if it doesn't exist yet - if os.path.exists(fullPath) and not overwrite: - raise FileExistsError(str(fullPath)) - with open(fullPath, "wb+") as file: + if Path.exists(filePath) and not overwrite: + raise FileExistsError(str(filePath)) + with open(filePath, "wb+") as file: file.write(response.content) diff --git a/src/onc/onc.py b/src/onc/onc.py index 1bd628b..7327ed9 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -30,15 +30,7 @@ def __init__( self.showInfo = showInfo self.timeout = timeout self.baseUrl = "https://data.oceannetworks.ca/" - self.outPath = "" - - outPath = str(outPath) - # sanitize outPath - if len(outPath) > 0: - outPath = outPath.replace("\\", "/") - if outPath[-1] == "/": - outPath = outPath[:-1] - self.outPath = outPath + self.outPath = Path(outPath) # switch to qa if needed if not production: @@ -53,13 +45,18 @@ def __init__( def print(self, obj, filename: str = ""): """ Helper for printing a JSON dictionary to the console or to a file - @filename: if present, creates the file and writes the output in it + @filename: if present, creates a file with a ".json" extension + in "self.outPath" directory, and writes the output to the file. + if not present, prints the output to the console. """ text = json.dumps(obj, indent=4) if filename == "": print(text) else: - with open(filename, "w+") as file: + filePath = self.outPath / filename + filePath = filePath.with_suffix(".json") + + with open(filePath, "w+") as file: file.write(text) def formatUtc(self, dateString: str = "now"): diff --git a/tests/README.md b/tests/README.md index 4bf484b..e552748 100644 --- a/tests/README.md +++ b/tests/README.md @@ -1,84 +1,94 @@ # Testing Documentation -This directory contains an automated test suite written for the Python API client using +This directory contains an automated test suite written for the Python API client using the [Robot Framework](http://robotframework.org) (RF from now on) as well as pytest. Directory structure is as follows: -* libraries: Python 3 library files used in the tests -* output: Default output directory for methods that download files (i.e. orderDataProduct()) -* pytests: Tests converted from RF to pytest format -* resources: Robot generic scripts to be reused by tests -* suites: Test suites +- libraries: Python 3 library files used in the tests +- output: Default output directory for methods that download files (i.e. orderDataProduct()) +- pytests: Tests converted from RF to pytest format +- resources: Robot generic scripts to be reused by tests +- suites: Test suites Read the test suites (.robot files) in "suites" to understand what exactly is being tested. Test suites contain the test cases in the "Test Cases" section, and are written in a language similar to english. - ## Testing Requirements 1. Make sure Python 3 and pip are installed properly. It is highly suggested to use a virtual environment. 2. Install [Robot Framework](https://robotframework.org/) and [python-dotenv](https://saurabh-kumar.com/python-dotenv/) + ```shell pip install robotframework python-dotenv ``` + (or use "pip3" depending on your system configuration) 3. Optional: install [pabot](https://pabot.org/) for test parallelization: + ```shell pip install -U robotframework-pabot ``` + 4. Install this project in editable mode (assume the current directory is the root) + ```shell pip install -e . ``` - ## Running the Tests -In the terminal, run all tests from the root directory. +In the terminal, run all tests from the root directory. Tests can also be run from a different folder. Just change the relative path of the test suites. Create a `.env` file under tests folder and put TOKEN variable in the file. If you are on Windows, make sure the encoding of `.env` file is UTF-8 after using the command below. + ```shell echo TOKEN=${YOUR_TOKEN} > .env ``` -The default testing environment is PROD. If you are an internal developer, add the following line to .env so that the tests are running against QA. +The default testing environment is PROD. If you are an internal developer, add the following line to .env so that the tests are running against QA. ```shell echo ONC_ENV=QA >> .env ``` + Change ONC_ENV value from QA to PROD if testing in PROD is needed. Removing the line also does the trick. -*To run all the RF test suites (parallelized):* +_To run all the RF test suites (parallelized):_ + ```shell -pabot --testlevelsplit tests/suites +pabot --testlevelsplit tests/robot/suites ``` -*To run a single RF test suite (replace 0X with the prefix of the test file name, e.g., 01):* +_To run a single RF test suite (replace 0X with the prefix of the test file name, e.g., 01):_ + ```shell robot tests/suites/01* # robot tests/suites/0X* ``` -*To run a single RF test in a test suite (replace Y with the prefix of the test name, e.g., 01):* +_To run a single RF test in a test suite (replace Y with the prefix of the test name, e.g., 01):_ + ```shell robot --test "01*" tests/suites/01* # robot --test "Y*" tests/suites/0X* ``` -*To run pytest* +_To run pytest_ + ```shell pytest ``` -*`--variable TOKEN:${YOUR_TOKEN}` can be used if no `.env` file is present* +_`--variable TOKEN:${YOUR_TOKEN}` can be used if no `.env` file is present_ + ```shell robot --variable TOKEN:${YOUR_TOKEN} tests/suites/01* ``` Additionally, You can check the three bash files (testall, testcoverage and testsuite) for running the test suites. -Robot Framework also has plugins for IDEs like VS Code and Pycharm that makes running tests easier. +Robot Framework also has plugins for IDEs like VS Code and Pycharm that makes running tests easier. After tests finish, review the summary and logs in the root directory. @@ -92,18 +102,16 @@ just make the required changes, no coding knowledge is required. For anything more advanced than that, please read the Robot Framework Documentation and consider keeping the directory structure relevant. - ## Code Documentation Robot Framework promotes test cases written almost in plain english (if you need to document it, you're writing it wrong). -Still, code documentation is welcome if ever required. +Still, code documentation is welcome if ever required. If you are an internal user of Ocean Networks Canada, please refer to the [internal documentation page](https://internal.oceannetworks.ca/display/ONCData/11+-+Automated+User+Tests+for+API+Client+Libraries). - ## Acknowledgements Initial author: Dany Cabrera Maintainers: Kan Fu -Previous maintainers: Dany Cabrera \ No newline at end of file +Previous maintainers: Dany Cabrera diff --git a/tests/robot/libraries/delivery.py b/tests/robot/libraries/delivery.py index 6aaaba4..efa88f7 100644 --- a/tests/robot/libraries/delivery.py +++ b/tests/robot/libraries/delivery.py @@ -1,5 +1,7 @@ # delivery services' tests +from pathlib import Path + from common import onc @@ -15,7 +17,7 @@ def manualRunProduct(dpRequestId: int): def manualDownloadProduct(dpRunId: int, outPath: str = "", resultsOnly: bool = False): # Manually downloads runId - onc.outPath = outPath + onc.outPath = Path(outPath) return onc.downloadDataProduct(dpRunId, downloadResultsOnly=resultsOnly) diff --git a/tests/robot/suites/00__initial.robot b/tests/robot/suites/00__initial.robot index 40088c9..0932e74 100644 --- a/tests/robot/suites/00__initial.robot +++ b/tests/robot/suites/00__initial.robot @@ -20,5 +20,5 @@ Resource ../resources/general.robot Prepare output directory output/00/03 ${onc3}= Make ONC with path output/00/03 ${result}= Call Method ${onc3} getLocations ${F_LOCATIONCODE} - Call Method ${onc3} print ${result} output/00/03/03.json + Call Method ${onc3} print ${result} 03.json File Should Exist output/00/03/03.json \ No newline at end of file From 3d06c76915fde2e54a7fb209f908f14616873a4d Mon Sep 17 00:00:00 2001 From: Renfu Li Date: Wed, 20 Dec 2023 23:51:23 -0800 Subject: [PATCH 2/3] feat: add @property decorator to protect _out_path --- src/onc/modules/_util.py | 2 +- src/onc/onc.py | 17 +++++++++++++---- tests/README.md | 6 +++--- tests/robot/libraries/delivery.py | 2 +- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index 6b847d4..b5a4fc1 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -17,7 +17,7 @@ def saveAsFile( # Save file in outPath if it doesn't exist yet if Path.exists(filePath) and not overwrite: - raise FileExistsError(str(filePath)) + raise FileExistsError(filePath.resolve()) with open(filePath, "wb+") as file: file.write(response.content) diff --git a/src/onc/onc.py b/src/onc/onc.py index 7327ed9..0e09d55 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -30,9 +30,9 @@ def __init__( self.showInfo = showInfo self.timeout = timeout self.baseUrl = "https://data.oceannetworks.ca/" - self.outPath = Path(outPath) + self._out_path = Path(outPath) - # switch to qa if needed + # Switch to qa if needed if not production: self.baseUrl = "https://qa.oceannetworks.ca/" @@ -42,18 +42,27 @@ def __init__( self.realTime = _OncRealTime(self) self.archive = _OncArchive(self) + # Add getter and setter for self._out_path + @property + def outPath(self): + return self._out_path + + @outPath.setter + def outPath(self, outPath): + self._out_path = Path(outPath) + def print(self, obj, filename: str = ""): """ Helper for printing a JSON dictionary to the console or to a file @filename: if present, creates a file with a ".json" extension - in "self.outPath" directory, and writes the output to the file. + in "self._out_path" directory, and writes the output to the file. if not present, prints the output to the console. """ text = json.dumps(obj, indent=4) if filename == "": print(text) else: - filePath = self.outPath / filename + filePath = self._out_path / filename filePath = filePath.with_suffix(".json") with open(filePath, "w+") as file: diff --git a/tests/README.md b/tests/README.md index e552748..8caaaa7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -66,13 +66,13 @@ pabot --testlevelsplit tests/robot/suites _To run a single RF test suite (replace 0X with the prefix of the test file name, e.g., 01):_ ```shell -robot tests/suites/01* # robot tests/suites/0X* +robot tests/robot/suites/01* # robot tests/robot/suites/0X* ``` _To run a single RF test in a test suite (replace Y with the prefix of the test name, e.g., 01):_ ```shell -robot --test "01*" tests/suites/01* # robot --test "Y*" tests/suites/0X* +robot --test "01*" tests/robot/suites/01* # robot --test "Y*" tests/robot/suites/0X* ``` _To run pytest_ @@ -84,7 +84,7 @@ pytest _`--variable TOKEN:${YOUR_TOKEN}` can be used if no `.env` file is present_ ```shell -robot --variable TOKEN:${YOUR_TOKEN} tests/suites/01* +robot --variable TOKEN:${YOUR_TOKEN} tests/robot/suites/01* ``` Additionally, You can check the three bash files (testall, testcoverage and testsuite) for running the test suites. diff --git a/tests/robot/libraries/delivery.py b/tests/robot/libraries/delivery.py index efa88f7..82ca399 100644 --- a/tests/robot/libraries/delivery.py +++ b/tests/robot/libraries/delivery.py @@ -17,7 +17,7 @@ def manualRunProduct(dpRequestId: int): def manualDownloadProduct(dpRunId: int, outPath: str = "", resultsOnly: bool = False): # Manually downloads runId - onc.outPath = Path(outPath) + onc.outPath = outPath return onc.downloadDataProduct(dpRunId, downloadResultsOnly=resultsOnly) From 58c6d5dd96182812fdc7f665da5c4257dc4f3d34 Mon Sep 17 00:00:00 2001 From: Renfu Li Date: Tue, 2 Jan 2024 13:35:24 -0800 Subject: [PATCH 3/3] feat: fully resolve outPath for cleaner error messages --- src/onc/modules/_util.py | 2 +- src/onc/onc.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index b5a4fc1..66a05ba 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -17,7 +17,7 @@ def saveAsFile( # Save file in outPath if it doesn't exist yet if Path.exists(filePath) and not overwrite: - raise FileExistsError(filePath.resolve()) + raise FileExistsError(filePath) with open(filePath, "wb+") as file: file.write(response.content) diff --git a/src/onc/onc.py b/src/onc/onc.py index 0e09d55..1eb6533 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -30,7 +30,7 @@ def __init__( self.showInfo = showInfo self.timeout = timeout self.baseUrl = "https://data.oceannetworks.ca/" - self._out_path = Path(outPath) + self._out_path = Path(outPath).resolve() # Switch to qa if needed if not production: @@ -49,7 +49,7 @@ def outPath(self): @outPath.setter def outPath(self, outPath): - self._out_path = Path(outPath) + self._out_path = Path(outPath).resolve() def print(self, obj, filename: str = ""): """