From 436f9ce8e50b50a583720d90aa143210b2d5f506 Mon Sep 17 00:00:00 2001 From: Benjamin Hugo Date: Thu, 29 Sep 2022 14:55:34 +0200 Subject: [PATCH] Fix test case (#88) Fix longstanding test case failure - fix google drive downloading using gdrive ... stupidly google has stopped working for plain requests - migrate away from google drive downloading on the test server - pin numpy to <=1.19.5 -- newer numpy has shown numerical problems elsewhere - switch to py3.6 and higher - Add Python3.8 dockerfile and test --- HISTORY.rst | 6 ++++ Jenkinsfile.sh | 34 ++++++++++++++++++- docker/python36.docker | 5 +-- docker/python38.docker | 19 +++++++++++ setup.py | 9 ++++-- tricolour/tests/test_acceptance.py | 52 ++++++------------------------ 6 files changed, 75 insertions(+), 50 deletions(-) create mode 100644 docker/python38.docker diff --git a/HISTORY.rst b/HISTORY.rst index 8da1d40..6d81bce 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,12 @@ History ======= +0.1.9 (2022-09-29) +-------------------- +* Fix failing test case (need to switch to gdrive) +* Migrate off Google Drive downloads for CI server +* Pin numpy to remove deprecation errors (and possible numerical issues introduced in 1.20+) +* Pin pytest flake to < 5.0.0 to work around tholo/pytest-flake8#87 0.1.8 (2021-02-24) ------------------ diff --git a/Jenkinsfile.sh b/Jenkinsfile.sh index dbcff87..c72fb01 100644 --- a/Jenkinsfile.sh +++ b/Jenkinsfile.sh @@ -1,9 +1,41 @@ set -e set -u +set -x WORKSPACE_ROOT="$WORKSPACE/$BUILD_NUMBER" +echo "Setting up build in $WORKSPACE_ROOT" +TEST_OUTPUT_DIR_REL=testcase_output +TEST_OUTPUT_DIR="$WORKSPACE_ROOT/$TEST_OUTPUT_DIR_REL" +mkdir $TEST_OUTPUT_DIR +TEST_DATA_DIR="$WORKSPACE/../../../test-data" + # build docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python36.docker -t tricolour.1804.py36:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ +docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python38.docker -t tricolour.2004.py38:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/ #run tests -docker run --rm tricolour.1804.py36:${BUILD_NUMBER} +tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR +TEST_MS_REL=1519747221.subset.ms + +# test 3.6 +docker run \ + --rm \ + -v $TEST_OUTPUT_DIR:/testdata \ + --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \ + --workdir /code \ + --entrypoint /bin/sh \ + tricolour.1804.py36:${BUILD_NUMBER} \ + -c "python3 -m pytest --flake8 -s -vvv ." + +rm -rf $TEST_OUTPUT_DIR/TEST_MS_REL +tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR + +# test 3.8 +docker run \ + --rm \ + -v $TEST_OUTPUT_DIR:/testdata \ + --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \ + --workdir /code \ + --entrypoint /bin/sh \ + tricolour.2004.py38:${BUILD_NUMBER} \ + -c "python3 -m pytest --flake8 -s -vvv ." \ No newline at end of file diff --git a/docker/python36.docker b/docker/python36.docker index e5ecf36..d998e3c 100644 --- a/docker/python36.docker +++ b/docker/python36.docker @@ -1,4 +1,4 @@ -FROM kernsuite/base:5 +FROM kernsuite/base:6 # Install base requirements RUN docker-apt-install python3-pip \ @@ -17,6 +17,3 @@ WORKDIR /code # Install base + testing packages RUN pip3 install .[testing] - -# Run tests and flake at the same time -RUN py.test --flake8 -s -vvv . diff --git a/docker/python38.docker b/docker/python38.docker new file mode 100644 index 0000000..6221771 --- /dev/null +++ b/docker/python38.docker @@ -0,0 +1,19 @@ +FROM kernsuite/base:7 + +# Install base requirements +RUN docker-apt-install python3-pip \ + build-essential \ + cmake \ + casacore-dev \ + python-numpy \ + python-setuptools \ + libboost-python-dev \ + libcfitsio-dev \ + wcslib-dev + +RUN pip3 install --upgrade pip setuptools +ADD . /code +WORKDIR /code + +# Install base + testing packages +RUN pip3 install .[testing] diff --git a/setup.py b/setup.py index 0a78178..47ff136 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ requirements = [ 'dask[array] == 2021.2.0', 'donfig >= 0.4.0', - 'numpy >= 1.14.0', + 'numpy >= 1.14.0, <= 1.19.5', # breakage in newer numpy + numerical errors 'numba >= 0.43.0', 'scipy >= 1.2.0', 'threadpoolctl >= 1.0.0', @@ -19,7 +19,10 @@ 'zarr >= 2.3.1' ] -extras_require = {'testing': ['pytest', 'pytest-flake8', 'requests']} +extras_require = {'testing': ['pytest', + 'pytest-flake8', + 'flake8 >= 4.0.0, <5.0.0', + 'requests', 'gdown']} setup( author="Simon Perkins", @@ -44,7 +47,7 @@ keywords='tricolour', name='tricolour', packages=find_packages(), - python_requires=">=3.5", + python_requires=">=3.6", url='https://github.com/ska-sa/tricolour', version='0.1.8', zip_safe=False, diff --git a/tricolour/tests/test_acceptance.py b/tricolour/tests/test_acceptance.py index 41d971b..c3641a1 100644 --- a/tricolour/tests/test_acceptance.py +++ b/tricolour/tests/test_acceptance.py @@ -14,43 +14,16 @@ from pyrap.tables import table as tbl import numpy as np -import requests +import gdown import pytest _GOOGLE_FILE_ID = "1yxDIXUo3Xun9WXxA0x_hvX9Fmxo9Igpr" _MS_FILENAME = '1519747221.subset.ms' -def _get_confirm_token(response): - for key, value in response.cookies.items(): - if key.startswith('download_warning'): - return value - - return None - - -def _save_response_content(response, destination): - CHUNK_SIZE = 32768 - - with open(destination, "wb") as f: - for chunk in response.iter_content(CHUNK_SIZE): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - - def _download_file_from_google_drive(id, destination): - URL = "https://docs.google.com/uc?export=download" - - session = requests.Session() - - response = session.get(URL, params={'id': id}, stream=True) - token = _get_confirm_token(response) - - if token: - params = {'id': id, 'confirm': token} - response = session.get(URL, params=params, stream=True) - - _save_response_content(response, destination) + URL = f"https://drive.google.com/uc?id={_GOOGLE_FILE_ID}" + gdown.download(URL, destination, quiet=False) # Set timeout to 6 minutes @@ -59,23 +32,18 @@ def flagged_ms(request, tmp_path_factory): """ fixture yielding an MS flagged by tricolour """ + tmp_path = str(tmp_path_factory.mktemp('data')) + test_directory = os.path.dirname(__file__) try: - tarred_ms_filename = os.environ["TRICOLOUR_TEST_MS"] + ms_filename = os.environ["TRICOLOUR_TEST_MS"] except KeyError: tar_dir = tmp_path_factory.mktemp("tar-download") tarred_ms_filename = os.path.join(tar_dir, "test_data.tar.gz") - _download_file_from_google_drive(_GOOGLE_FILE_ID, tarred_ms_filename) - - tmp_path = str(tmp_path_factory.mktemp('data')) - - # Open and extract tarred ms - tarred_ms = tarfile.open(tarred_ms_filename) - tarred_ms.extractall(tmp_path) - - # Set up our paths - ms_filename = pjoin(tmp_path, _MS_FILENAME) - test_directory = os.path.dirname(__file__) + # Open and extract tarred ms + tarred_ms = tarfile.open(tarred_ms_filename) + tarred_ms.extractall(tmp_path) + ms_filename = pjoin(tmp_path, _MS_FILENAME) args = ['tricolour', '-fs', 'total_power',