From 74b44217a66199fa2e0f8e036955fc00f5cbc21a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 22 Feb 2024 20:27:28 +0530 Subject: [PATCH] Build and test PyWavelets Pyodide wheels in CI (#701) The main change is adding a Pyodide CI job. Other changes: * Add some relevant files and venvs to ignore * ensure file handles get closed and use importlib.resources instead of `__file__` * cache data loading * Convert `ecg` and `sst_nino3` data files from .npy to .npz (NumPy compressed archive) for Pyodide compatibility * Properly skip tests for WASM and wherever threading isn't availables * import `importlib.resources`, mark `/pywt/data/` as constant Co-authored-by: Ralf Gommers --- .github/workflows/emscripten.yml | 60 +++++++++++++++++++++ .gitignore | 8 +++ README.rst | 2 +- demo/batch_processing.py | 3 +- pywt/_pytest.py | 16 ++++-- pywt/data/_readers.py | 35 ++++++++---- pywt/data/{ecg.npy => ecg.npz} | Bin 4176 -> 4358 bytes pywt/data/{sst_nino3.npy => sst_nino3.npz} | Bin 64128 -> 64262 bytes pywt/tests/test_concurrent.py | 4 +- 9 files changed, 109 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/emscripten.yml rename pywt/data/{ecg.npy => ecg.npz} (93%) rename pywt/data/{sst_nino3.npy => sst_nino3.npz} (99%) diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 000000000..fcc906b37 --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,60 @@ +name: Test Pyodide build for PyWavelets + +on: + push: + branches: + - master + - v1.** + pull_request: + branches: + - master + - v1.** + +env: + FORCE_COLOR: 3 + +jobs: + build_wasm_emscripten: + name: Build PyWavelets for Pyodide + runs-on: ubuntu-latest + # Uncomment the following line to test changes on a fork + # if: github.repository == 'PyWavelets/pywt' + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: '3.11.2' + + - name: Install prerequisites + run: | + python -m pip install pyodide-build "pydantic<2" + echo EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) >> $GITHUB_ENV + + - name: Set up Emscripten toolchain + uses: mymindstorm/setup-emscripten@v14 + with: + version: ${{ env.EMSCRIPTEN_VERSION }} + actions-cache-folder: emsdk-cache + + - name: Set up Node.js + uses: actions/setup-node@v4.0.2 + with: + node-version: '18' + + - name: Build PyWavelets + run: | + pyodide build + + - name: Install and test wheel + run: | + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + pip install dist/*.whl + pushd demo + pip install matplotlib pytest + python -c "import pywt; print(pywt.__version__)" + pytest --pyargs pywt diff --git a/.gitignore b/.gitignore index d1c72e2d9..1bd984aa3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ __pycache__ *.py[co] *.pyd *.so +.DS_Store +.pytest_cache/ # Packages *.egg @@ -32,6 +34,12 @@ cythonize.dat pywt/version.py build.log +# Virtual environments +.env/ +env/ +venv/ +.venv/ + # asv files asv/env asv/html diff --git a/README.rst b/README.rst index 79f24f583..fd1aec91c 100644 --- a/README.rst +++ b/README.rst @@ -65,7 +65,7 @@ For more usage examples see the `demo`_ directory in the source package. Installation ------------ -PyWavelets supports `Python`_ >=3.7, and is only dependent on `NumPy`_ +PyWavelets supports `Python`_ >=3.9, and is only dependent on `NumPy`_ (supported versions are currently ``>= 1.14.6``). To pass all of the tests, `Matplotlib`_ is also required. `SciPy`_ is also an optional dependency. When present, FFT-based continuous wavelet transforms will use FFTs from SciPy diff --git a/demo/batch_processing.py b/demo/batch_processing.py index 1f55acc20..ae11a9fc8 100644 --- a/demo/batch_processing.py +++ b/demo/batch_processing.py @@ -24,8 +24,7 @@ from concurrent import futures except ImportError: raise ImportError( - "This demo requires concurrent.futures. It can be installed for " - "for python 2.x via: pip install futures") + "This demo requires concurrent.futures. If you are on WebAssembly, this is not available.") import numpy as np from numpy.testing import assert_array_equal diff --git a/pywt/_pytest.py b/pywt/_pytest.py index cfc9f0590..af4d60cf8 100644 --- a/pywt/_pytest.py +++ b/pywt/_pytest.py @@ -1,6 +1,7 @@ """common test-related code.""" import os import sys +import platform import multiprocessing import numpy as np import pytest @@ -18,15 +19,18 @@ ] try: - if sys.version_info[0] == 2: - import futures - else: - from concurrent import futures + from concurrent import futures max_workers = multiprocessing.cpu_count() futures_available = True except ImportError: futures_available = False futures = None + max_workers = 1 + +# Check if running on Emscripten/WASM, and skip tests that require concurrency. +# Relevant issue: https://github.com/pyodide/pyodide/issues/237 +IS_WASM = (sys.platform == "emscripten") or (platform.machine() in ["wasm32", "wasm64"]) + # check if pymatbridge + MATLAB tests should be run matlab_result_dict_dwt = None @@ -57,7 +61,9 @@ matlab_result_dict_dwt = np.load(matlab_data_file_dwt) uses_futures = pytest.mark.skipif( - not futures_available, reason='futures not available') + not futures_available or IS_WASM, + reason='futures is not available, or running via Pyodide/WASM.') + # not futures_available, reason='futures not available') uses_matlab = pytest.mark.skipif( matlab_missing, reason='pymatbridge and/or Matlab not available') uses_pymatbridge = pytest.mark.skipif( diff --git a/pywt/data/_readers.py b/pywt/data/_readers.py index 258230c20..10105a691 100644 --- a/pywt/data/_readers.py +++ b/pywt/data/_readers.py @@ -1,8 +1,14 @@ +import functools +import importlib.resources import os import numpy as np +_DATADIR = importlib.resources.files('pywt.data') + + +@functools.cache def ascent(): """ Get an 8-bit grayscale bit-depth, 512 x 512 derived image for @@ -36,11 +42,13 @@ def ascent(): >>> plt.show() # doctest: +SKIP """ - fname = os.path.join(os.path.dirname(__file__), 'ascent.npz') - ascent = np.load(fname)['data'] + with importlib.resources.as_file(_DATADIR.joinpath('ascent.npz')) as f: + ascent = np.load(f)['data'] + return ascent +@functools.cache def aero(): """ Get an 8-bit grayscale bit-depth, 512 x 512 derived image for @@ -71,11 +79,13 @@ def aero(): >>> plt.show() # doctest: +SKIP """ - fname = os.path.join(os.path.dirname(__file__), 'aero.npz') - aero = np.load(fname)['data'] + with importlib.resources.as_file(_DATADIR.joinpath('aero.npz')) as f: + aero = np.load(f)['data'] + return aero +@functools.cache def camera(): """ Get an 8-bit grayscale bit-depth, 512 x 512 derived image for @@ -117,11 +127,13 @@ def camera(): >>> plt.show() # doctest: +SKIP """ - fname = os.path.join(os.path.dirname(__file__), 'camera.npz') - camera = np.load(fname)['data'] + with importlib.resources.as_file(_DATADIR.joinpath('camera.npz')) as f: + camera = np.load(f)['data'] + return camera +@functools.cache def ecg(): """ Get 1024 points of an ECG timeseries. @@ -147,11 +159,13 @@ def ecg(): [] >>> plt.show() # doctest: +SKIP """ - fname = os.path.join(os.path.dirname(__file__), 'ecg.npy') - ecg = np.load(fname) + with importlib.resources.as_file(_DATADIR.joinpath('ecg.npz')) as f: + ecg = np.load(f)['data'] + return ecg +@functools.cache def nino(): """ This data contains the averaged monthly sea surface temperature in degrees @@ -183,8 +197,9 @@ def nino(): [] >>> plt.show() # doctest: +SKIP """ - fname = os.path.join(os.path.dirname(__file__), 'sst_nino3.npy') - sst_csv = np.load(fname) + with importlib.resources.as_file(_DATADIR.joinpath('sst_nino3.npz')) as f: + sst_csv = np.load(f)['data'] + # sst_csv = pd.read_csv("http://www.cpc.ncep.noaa.gov/data/indices/ersst4.nino.mth.81-10.ascii", sep=' ', skipinitialspace=True) # take only full years n = int(np.floor(sst_csv.shape[0]/12.)*12.) diff --git a/pywt/data/ecg.npy b/pywt/data/ecg.npz similarity index 93% rename from pywt/data/ecg.npy rename to pywt/data/ecg.npz index 119916b039439aef044ae8a4ab1b0d9cb1de3a2a..ac3961a2da271468d7a34a626a2736f12a3230fb 100644 GIT binary patch delta 196 zcmcbh(5B=Z;LXgU%K!n23J$*wzz8B?P6+U3WdrdUfzS*{?*cmq E040bd#{d8T delta 21 ccmZoux}d;0*)P;LAd->6ZKAW=#+{4;089!7ssI20 diff --git a/pywt/data/sst_nino3.npy b/pywt/data/sst_nino3.npz similarity index 99% rename from pywt/data/sst_nino3.npy rename to pywt/data/sst_nino3.npz index 822d2021b6bd1766d852c018414e54981003187e..60d1d5856ed91fba82c8eb8c0e75d45b01b06785 100644 GIT binary patch delta 145 zcmZqp%G~yi*($)BnMIcY0u&icE2@?MLjeba2t!I@Nupj}K_w%D07Jtsh$