Fix test case (#88)

Fix longstanding test case failure

- fix google drive downloading using gdrive ... stupidly google has stopped working for plain requests
- migrate away from google drive downloading on the test server
- pin numpy to <=1.19.5 -- newer numpy has shown numerical problems elsewhere
- switch to py3.6 and higher
- Add Python3.8 dockerfile and test

HISTORY.rst

@@ -2,6 +2,12 @@
 History
 =======
 
+0.1.9 (2022-09-29)
+--------------------
+* Fix failing test case (need to switch to gdrive)
+* Migrate off Google Drive downloads for CI server
+* Pin numpy to remove deprecation errors (and possible numerical issues introduced in 1.20+)
+* Pin pytest flake to < 5.0.0 to work around tholo/pytest-flake8#87
 
 0.1.8 (2021-02-24)
 ------------------

Jenkinsfile.sh

@@ -1,9 +1,41 @@
 set -e
 set -u
+set -x
 WORKSPACE_ROOT="$WORKSPACE/$BUILD_NUMBER"
+echo "Setting up build in $WORKSPACE_ROOT"
+TEST_OUTPUT_DIR_REL=testcase_output
+TEST_OUTPUT_DIR="$WORKSPACE_ROOT/$TEST_OUTPUT_DIR_REL"
+mkdir $TEST_OUTPUT_DIR
+TEST_DATA_DIR="$WORKSPACE/../../../test-data"
+
 
 # build
 docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python36.docker -t tricolour.1804.py36:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/
+docker build -f ${WORKSPACE_ROOT}/projects/tricolour/docker/python38.docker -t tricolour.2004.py38:${BUILD_NUMBER} ${WORKSPACE_ROOT}/projects/tricolour/
 
 #run tests
-docker run --rm tricolour.1804.py36:${BUILD_NUMBER}
+tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR
+TEST_MS_REL=1519747221.subset.ms
+
+# test 3.6
+docker run \
+    --rm \
+    -v $TEST_OUTPUT_DIR:/testdata \
+    --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \
+    --workdir /code \
+    --entrypoint /bin/sh \
+    tricolour.1804.py36:${BUILD_NUMBER} \
+    -c "python3 -m pytest --flake8 -s -vvv ."
+
+rm -rf $TEST_OUTPUT_DIR/TEST_MS_REL
+tar xvf $TEST_DATA_DIR/acceptance_test_data.tar.gz -C $TEST_OUTPUT_DIR
+
+# test 3.8
+docker run \
+    --rm \
+    -v $TEST_OUTPUT_DIR:/testdata \
+    --env TRICOLOUR_TEST_MS=/testdata/$TEST_MS_REL \
+    --workdir /code \
+    --entrypoint /bin/sh \
+    tricolour.2004.py38:${BUILD_NUMBER} \
+    -c "python3 -m pytest --flake8 -s -vvv ."

docker/python36.docker

@@ -1,4 +1,4 @@
-FROM kernsuite/base:5
+FROM kernsuite/base:6
 
 # Install base requirements
 RUN docker-apt-install python3-pip \
@@ -17,6 +17,3 @@ WORKDIR /code
 
 # Install base + testing packages
 RUN pip3 install .[testing]
-
-# Run tests and flake at the same time
-RUN py.test --flake8 -s -vvv .

docker/python38.docker

@@ -0,0 +1,19 @@
+FROM kernsuite/base:7
+
+# Install base requirements
+RUN docker-apt-install python3-pip \
+                       build-essential \
+                       cmake \
+                       casacore-dev \
+                       python-numpy \
+                       python-setuptools \
+                       libboost-python-dev \
+                       libcfitsio-dev \
+                       wcslib-dev
+
+RUN pip3 install --upgrade pip setuptools
+ADD . /code
+WORKDIR /code
+
+# Install base + testing packages
+RUN pip3 install .[testing]

setup.py

@@ -11,15 +11,18 @@
 requirements = [
     'dask[array] == 2021.2.0',
     'donfig >= 0.4.0',
-    'numpy >= 1.14.0',
+    'numpy >= 1.14.0, <= 1.19.5',  # breakage in newer numpy + numerical errors
     'numba >= 0.43.0',
     'scipy >= 1.2.0',
     'threadpoolctl >= 1.0.0',
     'dask-ms == 0.2.6',
     'zarr >= 2.3.1'
 ]
 
-extras_require = {'testing': ['pytest', 'pytest-flake8', 'requests']}
+extras_require = {'testing': ['pytest',
+                              'pytest-flake8',
+                              'flake8 >= 4.0.0, <5.0.0',
+                              'requests', 'gdown']}
 
 setup(
     author="Simon Perkins",
@@ -44,7 +47,7 @@
     keywords='tricolour',
     name='tricolour',
     packages=find_packages(),
-    python_requires=">=3.5",
+    python_requires=">=3.6",
     url='https://github.com/ska-sa/tricolour',
     version='0.1.8',
     zip_safe=False,

tricolour/tests/test_acceptance.py

@@ -14,43 +14,16 @@
 
 from pyrap.tables import table as tbl
 import numpy as np
-import requests
+import gdown
 import pytest
 
 _GOOGLE_FILE_ID = "1yxDIXUo3Xun9WXxA0x_hvX9Fmxo9Igpr"
 _MS_FILENAME = '1519747221.subset.ms'
 
 
-def _get_confirm_token(response):
-    for key, value in response.cookies.items():
-        if key.startswith('download_warning'):
-            return value
-
-    return None
-
-
-def _save_response_content(response, destination):
-    CHUNK_SIZE = 32768
-
-    with open(destination, "wb") as f:
-        for chunk in response.iter_content(CHUNK_SIZE):
-            if chunk:  # filter out keep-alive new chunks
-                f.write(chunk)
-
-
 def _download_file_from_google_drive(id, destination):
-    URL = "https://docs.google.com/uc?export=download"
-
-    session = requests.Session()
-
-    response = session.get(URL, params={'id': id}, stream=True)
-    token = _get_confirm_token(response)
-
-    if token:
-        params = {'id': id, 'confirm': token}
-        response = session.get(URL, params=params, stream=True)
-
-    _save_response_content(response, destination)
+    URL = f"https://drive.google.com/uc?id={_GOOGLE_FILE_ID}"
+    gdown.download(URL, destination, quiet=False)
 
 
 # Set timeout to 6 minutes
@@ -59,23 +32,18 @@ def flagged_ms(request, tmp_path_factory):
     """
     fixture yielding an MS flagged by tricolour
     """
+    tmp_path = str(tmp_path_factory.mktemp('data'))
+    test_directory = os.path.dirname(__file__)
     try:
-        tarred_ms_filename = os.environ["TRICOLOUR_TEST_MS"]
+        ms_filename = os.environ["TRICOLOUR_TEST_MS"]
     except KeyError:
         tar_dir = tmp_path_factory.mktemp("tar-download")
         tarred_ms_filename = os.path.join(tar_dir, "test_data.tar.gz")
-
         _download_file_from_google_drive(_GOOGLE_FILE_ID, tarred_ms_filename)
-
-    tmp_path = str(tmp_path_factory.mktemp('data'))
-
-    # Open and extract tarred ms
-    tarred_ms = tarfile.open(tarred_ms_filename)
-    tarred_ms.extractall(tmp_path)
-
-    # Set up our paths
-    ms_filename = pjoin(tmp_path, _MS_FILENAME)
-    test_directory = os.path.dirname(__file__)
+        # Open and extract tarred ms
+        tarred_ms = tarfile.open(tarred_ms_filename)
+        tarred_ms.extractall(tmp_path)
+        ms_filename = pjoin(tmp_path, _MS_FILENAME)
 
     args = ['tricolour',
             '-fs', 'total_power',