From da8ee2ce902b6e9a48bc6566fa69d2a3dadb9dba Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 14 Aug 2024 16:10:17 -0400 Subject: [PATCH 01/20] Added test-emscripten dependencies --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 0a5b74418..28e0ffa96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,6 +80,21 @@ test = [ "scikit-hep-testdata", "rangehttpserver" ] +test-emscripten = [ + "xxhash", + "minio", + "aiohttp", + "fsspec-xrootd", + "s3fs", + "paramiko", + "pytest>=6", + "pytest-timeout", + "pytest-rerunfailures", + "requests", + "scikit-hep-testdata", + "rangehttpserver", + "pyodide-http" +] xrootd = ["fsspec-xrootd"] [project.urls] From f68c18f1be1963db5adfe8d2730a7112b527a85f Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 14 Aug 2024 16:17:44 -0400 Subject: [PATCH 02/20] Added emscripten-build workflow --- .github/workflows/build-test.yml | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 6248c0dd5..8968d8953 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -116,3 +116,56 @@ jobs: - name: Run pytest run: | python -m pytest -vv tests --reruns 10 --reruns-delay 30 --only-rerun "(?i)http|ssl|timeout|expired|connection|socket" + + emscripten-build: + name: Build and test with Emscripten + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install pyodide-build + run: python3 -m pip install pyodide-build + + - name: Determine EMSDK version + id: compute-emsdk-version + run: | + pyodide config list + # Save EMSDK version + EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version) + echo "emsdk-version=$EMSCRIPTEN_VERSION" >> $GITHUB_OUTPUT + + - name: Install EMSDK + uses: mymindstorm/setup-emsdk@v14 + with: + version: ${{ steps.compute-emsdk-version.outputs.emsdk-version }} + + - name: Build the package + run: pyodide build + + - name: Set up Pyodide virtual environment + run: | + pyodide venv .venv-pyodide + source .venv-pyodide/bin/activate + # clone and install scikit-hep-testdata so that test files are available locally + git clone https://github.com/scikit-hep/scikit-hep-testdata.git + python -m pip install -e scikit-hep-testdata + python -m pip install .[test-emscripten] + + - name: Run pytest + run: | + source .venv-pyodide/bin/activate + # run a few tests compatible with wasm + # cache and timeout don't work in wasm so we disable them + python -m pytest -p no:cacheprovider --timeout 0 -vv \ + tests/test_0014_all_ttree_versions.py \ + tests/test_0016_interpretations.py \ + tests/test_0962_RNTuple_update.py \ + tests/test_1191_rntuple_fixes.py From c9cf45d2fab39bbfbd4fc0900cc47fd8e33a83af Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 11:46:53 -0400 Subject: [PATCH 03/20] Ignore Pyodide files --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 0aceed8b2..777961eb1 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,10 @@ dmypy.json # Pyre type checker .pyre/ + +# Local copies of skhep_testdata files +skhep_testdata/ + +# Pyodide +.pyodide* +dist-pyodide/ From bbaf228c842e74712b664b6dced0ea9f4b75949a Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 11:47:52 -0400 Subject: [PATCH 04/20] Updated dependencies to run pyodide tests --- pyproject.toml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 28e0ffa96..1a38dde6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,20 +80,8 @@ test = [ "scikit-hep-testdata", "rangehttpserver" ] -test-emscripten = [ - "xxhash", - "minio", - "aiohttp", - "fsspec-xrootd", - "s3fs", - "paramiko", - "pytest>=6", - "pytest-timeout", - "pytest-rerunfailures", - "requests", - "scikit-hep-testdata", - "rangehttpserver", - "pyodide-http" +test-pyodide = [ + "pytest-pyodide" ] xrootd = ["fsspec-xrootd"] From ffef578c8a51bf4c3d376725879392971a88f86f Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 11:48:59 -0400 Subject: [PATCH 05/20] Updated workflow to run Pyodide tests --- .github/workflows/build-test.yml | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 8968d8953..03c0a88d1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -117,8 +117,7 @@ jobs: run: | python -m pytest -vv tests --reruns 10 --reruns-delay 30 --only-rerun "(?i)http|ssl|timeout|expired|connection|socket" - emscripten-build: - name: Build and test with Emscripten + pyodide-build: runs-on: ubuntu-latest timeout-minutes: 30 @@ -132,7 +131,7 @@ jobs: python-version: '3.12' - name: Install pyodide-build - run: python3 -m pip install pyodide-build + run: python3 -m pip install pyodide-build==0.28.0 - name: Determine EMSDK version id: compute-emsdk-version @@ -150,22 +149,16 @@ jobs: - name: Build the package run: pyodide build - - name: Set up Pyodide virtual environment + - name: Build an awkward wheel compatible with the awkward-cpp version in pyodide run: | - pyodide venv .venv-pyodide - source .venv-pyodide/bin/activate - # clone and install scikit-hep-testdata so that test files are available locally - git clone https://github.com/scikit-hep/scikit-hep-testdata.git - python -m pip install -e scikit-hep-testdata - python -m pip install .[test-emscripten] + git clone --depth 1 --branch v2.6.4 https://github.com/scikit-hep/awkward.git dependencies/awkward + pyodide build dependencies/awkward + rm -rf dependencies/ - name: Run pytest - run: | - source .venv-pyodide/bin/activate - # run a few tests compatible with wasm - # cache and timeout don't work in wasm so we disable them - python -m pytest -p no:cacheprovider --timeout 0 -vv \ - tests/test_0014_all_ttree_versions.py \ - tests/test_0016_interpretations.py \ - tests/test_0962_RNTuple_update.py \ - tests/test_1191_rntuple_fixes.py + uses: pyodide/pytest-pyodide/.github/workflows/main.yaml@main + with: + build-artifact-name: none + build-artifact-path: dist + pyodide-version: 0.26.2 + pytest-extra-args: -vv tests-wasm From 88ca771ea8952a3d4a8945a84301c2dc8fc06109 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 11:53:58 -0400 Subject: [PATCH 06/20] Added simple Pyodide test --- tests-wasm/__init__.py | 1 + .../test_1272_basic_pyodide_functionality.py | 33 ++++++++++++ tests-wasm/utils.py | 50 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 tests-wasm/__init__.py create mode 100644 tests-wasm/test_1272_basic_pyodide_functionality.py create mode 100644 tests-wasm/utils.py diff --git a/tests-wasm/__init__.py b/tests-wasm/__init__.py new file mode 100644 index 000000000..f8c1f9969 --- /dev/null +++ b/tests-wasm/__init__.py @@ -0,0 +1 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE diff --git a/tests-wasm/test_1272_basic_pyodide_functionality.py b/tests-wasm/test_1272_basic_pyodide_functionality.py new file mode 100644 index 000000000..09b09df04 --- /dev/null +++ b/tests-wasm/test_1272_basic_pyodide_functionality.py @@ -0,0 +1,33 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +import pytest + +try: + from utils import run_test_in_pyodide +except ImportError: + pytest.skip("Pyodide is not available", allow_module_level=True) + + +@run_test_in_pyodide(test_file="test_ntuple_extension_columns.root") +def test_schema_extension(selenium): + import uproot + + with uproot.open("test_ntuple_extension_columns.root") as f: + obj = f["EventData"] + + assert len(obj.column_records) > len(obj.header.column_records) + assert len(obj.column_records) == 936 + assert obj.column_records[903].first_ele_index == 36 + + arrays = obj.arrays() + + pbs = arrays[ + "HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux::fastDIPS20211215_pb" + ] + assert len(pbs) == 40 + assert all(len(a) == 0 for a in pbs[:36]) + assert next(i for i, a in enumerate(pbs) if len(a) != 0) == 36 + + jets = arrays["HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux:"] + assert len(jets.pt) == len(pbs) diff --git a/tests-wasm/utils.py b/tests-wasm/utils.py new file mode 100644 index 000000000..4fb6bce4c --- /dev/null +++ b/tests-wasm/utils.py @@ -0,0 +1,50 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +import os +import pathlib +import shutil +from functools import wraps + +import pytest +import skhep_testdata + +try: + from pytest_pyodide import run_in_pyodide + from pytest_pyodide.decorator import copy_files_to_pyodide +except ImportError: + pytest.skip("Pyodide is not available", allow_module_level=True) + + +# copy skhep_testdata files to testdata directory (needed for @copy_files_to_pyodide) +def ensure_testdata(filename): + if not pathlib.Path("skhep_testdata/" + filename).is_file(): + filepath = skhep_testdata.data_path(filename) + os.makedirs("skhep_testdata", exist_ok=True) + shutil.copyfile(filepath, "skhep_testdata/" + filename) + + +def run_test_in_pyodide(test_file=None): + def decorator(test_func): + @wraps(test_func) + def wrapper(selenium): + if test_file is not None: + ensure_testdata(test_file) + + @copy_files_to_pyodide( + file_list=[("dist", "dist")] + + ( + [] + if test_file is None + else [("skhep_testdata/" + test_file, test_file)] + ), + install_wheels=True, + ) + def inner_func(selenium): + run_in_pyodide()(test_func)(selenium) + + return inner_func(selenium) + + return wrapper + + return decorator From ef82fc173fbd6535e60dba2fbd02693f6bd660d5 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 12:05:04 -0400 Subject: [PATCH 07/20] Moved Pyodide testing to a separate job --- .github/workflows/build-test.yml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 03c0a88d1..5276b84e3 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -155,10 +155,18 @@ jobs: pyodide build dependencies/awkward rm -rf dependencies/ - - name: Run pytest - uses: pyodide/pytest-pyodide/.github/workflows/main.yaml@main + - name: Upload wheels artifact + uses: actions/upload-artifact@v3 with: - build-artifact-name: none - build-artifact-path: dist - pyodide-version: 0.26.2 - pytest-extra-args: -vv tests-wasm + name: pyodide_wheels + path: dist + + pyodide-test: + needs: pyodide-build + timeout-minutes: 30 + uses: pyodide/pytest-pyodide/.github/workflows/main.yaml@main + with: + build-artifact-name: pyodide_wheels + build-artifact-path: dist + pyodide-version: 0.26.2 + pytest-extra-args: -vv tests-wasm From 00e3949a3b4d35f54be8743307f811a70626974f Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 12:09:14 -0400 Subject: [PATCH 08/20] Removed timeout --- .github/workflows/build-test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 5276b84e3..eb34b6f8c 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -163,7 +163,6 @@ jobs: pyodide-test: needs: pyodide-build - timeout-minutes: 30 uses: pyodide/pytest-pyodide/.github/workflows/main.yaml@main with: build-artifact-name: pyodide_wheels From bd4532279f96f32725072e5fb7c51f31c9a6edca Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 13:21:11 -0400 Subject: [PATCH 09/20] Fixed Pyodide testing --- .github/workflows/build-test.yml | 40 ++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index eb34b6f8c..e16730df1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -120,6 +120,10 @@ jobs: pyodide-build: runs-on: ubuntu-latest timeout-minutes: 30 + env: + PYODIDE_VERSION: 0.26.2 + PYODIDE_BUILD_VERSION: 0.28.0 + AWKWARD_VERSION: v2.6.4 steps: - name: Checkout @@ -131,7 +135,7 @@ jobs: python-version: '3.12' - name: Install pyodide-build - run: python3 -m pip install pyodide-build==0.28.0 + run: python3 -m pip install pyodide-build==$PYODIDE_BUILD_VERSION - name: Determine EMSDK version id: compute-emsdk-version @@ -151,21 +155,27 @@ jobs: - name: Build an awkward wheel compatible with the awkward-cpp version in pyodide run: | - git clone --depth 1 --branch v2.6.4 https://github.com/scikit-hep/awkward.git dependencies/awkward + git clone --depth 1 --branch $AWKWARD_VERSION https://github.com/scikit-hep/awkward.git dependencies/awkward pyodide build dependencies/awkward rm -rf dependencies/ - - name: Upload wheels artifact - uses: actions/upload-artifact@v3 + - name: Download Pyodide + uses: pyodide/pyodide-actions/download-pyodide@v1 with: - name: pyodide_wheels - path: dist - - pyodide-test: - needs: pyodide-build - uses: pyodide/pytest-pyodide/.github/workflows/main.yaml@main - with: - build-artifact-name: pyodide_wheels - build-artifact-path: dist - pyodide-version: 0.26.2 - pytest-extra-args: -vv tests-wasm + version: ${{ env.PYODIDE_VERSION }} + to: pyodide-dist + + - name: Install browser + uses: pyodide/pyodide-actions/install-browser@v1 + with: + runner: selenium + browser: chrome + browser-version: latest + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install dependencies + run: pip install .[test-pyodide] pyodide-py==$PYODIDE_VERSION + + - name: Run pytest + run: | + python -m pytest -vv --dist-dir=./pyodide-dist/ --runner=selenium --rt=chrome tests-wasm From 7d34090e37305247fc34f9e4c2c41d1cde715267 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 13:37:30 -0400 Subject: [PATCH 10/20] Fixed test dependencies --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1a38dde6d..6febdcafb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,9 @@ test = [ "rangehttpserver" ] test-pyodide = [ - "pytest-pyodide" + "pytest>=6", + "pytest-pyodide", + "pytest-timeout" ] xrootd = ["fsspec-xrootd"] From 30aa991f4b374d6a9d1ae7edcd288cfa99eaf21a Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Mon, 26 Aug 2024 14:41:44 -0400 Subject: [PATCH 11/20] Fixed dependencies --- .github/workflows/build-test.yml | 2 +- pyproject.toml | 3 ++- tests-wasm/test_1272_basic_pyodide_functionality.py | 7 +------ 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e16730df1..7f1c664e1 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -178,4 +178,4 @@ jobs: - name: Run pytest run: | - python -m pytest -vv --dist-dir=./pyodide-dist/ --runner=selenium --rt=chrome tests-wasm + pytest -vv --dist-dir=./pyodide-dist/ --runner=selenium --rt=chrome tests-wasm diff --git a/pyproject.toml b/pyproject.toml index 6febdcafb..08cda684e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,8 @@ test = [ test-pyodide = [ "pytest>=6", "pytest-pyodide", - "pytest-timeout" + "pytest-timeout", + "scikit-hep-testdata" ] xrootd = ["fsspec-xrootd"] diff --git a/tests-wasm/test_1272_basic_pyodide_functionality.py b/tests-wasm/test_1272_basic_pyodide_functionality.py index 09b09df04..1a00778fe 100644 --- a/tests-wasm/test_1272_basic_pyodide_functionality.py +++ b/tests-wasm/test_1272_basic_pyodide_functionality.py @@ -1,12 +1,7 @@ # BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE from __future__ import annotations -import pytest - -try: - from utils import run_test_in_pyodide -except ImportError: - pytest.skip("Pyodide is not available", allow_module_level=True) +from utils import run_test_in_pyodide @run_test_in_pyodide(test_file="test_ntuple_extension_columns.root") From 5963f27842e0f723fc7d70363bbd4a8476be6bda Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Tue, 27 Aug 2024 09:30:39 -0400 Subject: [PATCH 12/20] Patched a few functions to work with pyodide --- src/uproot/source/coalesce.py | 13 ++++++++++++- src/uproot/source/fsspec.py | 11 +++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/uproot/source/coalesce.py b/src/uproot/source/coalesce.py index 52cad9289..824234535 100644 --- a/src/uproot/source/coalesce.py +++ b/src/uproot/source/coalesce.py @@ -6,6 +6,7 @@ from __future__ import annotations import queue +import sys from concurrent.futures import Future from dataclasses import dataclass from typing import Callable @@ -33,6 +34,9 @@ def add_done_callback(self, callback, *, context=None): self._parent.add_done_callback(callback) def result(self, timeout=None): + if sys.platform == "emscripten": + # Pyodide futures don't support timeout + return self._parent.result()[self._s] return self._parent.result(timeout=timeout)[self._s] @@ -126,7 +130,14 @@ def coalesce_requests( def chunkify(req: RangeRequest): chunk = uproot.source.chunk.Chunk(source, req.start, req.stop, req.future) - req.future.add_done_callback(uproot.source.chunk.notifier(chunk, notifications)) + if sys.platform == "emscripten": + # Callbacks don't work in pyodide yet, so we call the notifier directly + # TODO: Remove this once pyodide supports threads + uproot.source.chunk.notifier(chunk, notifications)() + else: + req.future.add_done_callback( + uproot.source.chunk.notifier(chunk, notifications) + ) return chunk return list(map(chunkify, all_requests)) diff --git a/src/uproot/source/fsspec.py b/src/uproot/source/fsspec.py index 588b6b31d..0ba1bc61e 100644 --- a/src/uproot/source/fsspec.py +++ b/src/uproot/source/fsspec.py @@ -5,6 +5,7 @@ import asyncio import concurrent.futures import queue +import sys import fsspec import fsspec.asyn @@ -164,6 +165,16 @@ def submit(request_ranges: list[tuple[int, int]]): self._fs.cat_ranges, paths=paths, starts=starts, ends=ends ) ) + if sys.platform == "emscripten": + # Threads can't be spawned in pyodide yet, so we run the function directly + # and return a future that is already resolved. + # TODO: remove this when pyodide supports threads + loop = asyncio.get_event_loop() + future = loop.create_future() + future.set_result( + self._fs.cat_ranges(paths=paths, starts=starts, ends=ends) + ) + return future return self._executor.submit(coroutine) return coalesce_requests( From a62e8017668e51335ff42afc813464d15eeb94b6 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Tue, 27 Aug 2024 09:41:55 -0400 Subject: [PATCH 13/20] Added more tests --- ....py => test_1272_basic_rntuple_reading.py} | 14 + tests-wasm/test_1272_basic_ttree_reading.py | 1147 +++++++++++++++++ 2 files changed, 1161 insertions(+) rename tests-wasm/{test_1272_basic_pyodide_functionality.py => test_1272_basic_rntuple_reading.py} (66%) create mode 100644 tests-wasm/test_1272_basic_ttree_reading.py diff --git a/tests-wasm/test_1272_basic_pyodide_functionality.py b/tests-wasm/test_1272_basic_rntuple_reading.py similarity index 66% rename from tests-wasm/test_1272_basic_pyodide_functionality.py rename to tests-wasm/test_1272_basic_rntuple_reading.py index 1a00778fe..cb86228ea 100644 --- a/tests-wasm/test_1272_basic_pyodide_functionality.py +++ b/tests-wasm/test_1272_basic_rntuple_reading.py @@ -26,3 +26,17 @@ def test_schema_extension(selenium): jets = arrays["HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux:"] assert len(jets.pt) == len(pbs) + + +@run_test_in_pyodide(test_file="Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root") +def test_split_encoding(): + import uproot + + with uproot.open("Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root") as f: + obj = f["Events"] + arrays = obj.arrays() + + expected_pt = [10.763696670532227, 15.736522674560547] + expected_charge = [-1, -1] + assert arrays["Muon_pt"][0].tolist() == expected_pt + assert arrays["Muon_charge"][0].tolist() == expected_charge diff --git a/tests-wasm/test_1272_basic_ttree_reading.py b/tests-wasm/test_1272_basic_ttree_reading.py new file mode 100644 index 000000000..153b75ae9 --- /dev/null +++ b/tests-wasm/test_1272_basic_ttree_reading.py @@ -0,0 +1,1147 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +from utils import run_test_in_pyodide + + +# Taken from test_0023_ttree_versions.py +@run_test_in_pyodide(test_file="uproot-sample-6.20.04-uncompressed.root") +def test_read_ttree(selenium): + truth = { + "n": [ + 0, + 1, + 2, + 3, + 4, + 0, + 1, + 2, + 3, + 4, + 0, + 1, + 2, + 3, + 4, + 0, + 1, + 2, + 3, + 4, + 0, + 1, + 2, + 3, + 4, + 0, + 1, + 2, + 3, + 4, + ], + "b": [ + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + ], + "ab": [ + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + [False, True, False], + [True, False, True], + ], + "Ab": [ + [], + [True], + [True, True], + [True, True, True], + [True, True, True, True], + [], + [False], + [False, False], + [False, False, False], + [False, False, False, False], + [], + [True], + [True, True], + [True, True, True], + [True, True, True, True], + [], + [False], + [False, False], + [False, False, False], + [False, False, False, False], + [], + [True], + [True, True], + [True, True, True], + [True, True, True, True], + [], + [False], + [False, False], + [False, False, False], + [False, False, False, False], + ], + "i1": [ + -15, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + ], + "ai1": [ + [-14, -13, -12], + [-13, -12, -11], + [-12, -11, -10], + [-11, -10, -9], + [-10, -9, -8], + [-9, -8, -7], + [-8, -7, -6], + [-7, -6, -5], + [-6, -5, -4], + [-5, -4, -3], + [-4, -3, -2], + [-3, -2, -1], + [-2, -1, 0], + [-1, 0, 1], + [0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + ], + "Ai1": [ + [], + [-15], + [-15, -13], + [-15, -13, -11], + [-15, -13, -11, -9], + [], + [-10], + [-10, -8], + [-10, -8, -6], + [-10, -8, -6, -4], + [], + [-5], + [-5, -3], + [-5, -3, -1], + [-5, -3, -1, 1], + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + ], + "u1": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + ], + "au1": [ + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + [16, 17, 18], + [17, 18, 19], + [18, 19, 20], + [19, 20, 21], + [20, 21, 22], + [21, 22, 23], + [22, 23, 24], + [23, 24, 25], + [24, 25, 26], + [25, 26, 27], + [26, 27, 28], + [27, 28, 29], + [28, 29, 30], + [29, 30, 31], + [30, 31, 32], + ], + "Au1": [ + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + [], + [15], + [15, 17], + [15, 17, 19], + [15, 17, 19, 21], + [], + [20], + [20, 22], + [20, 22, 24], + [20, 22, 24, 26], + [], + [25], + [25, 27], + [25, 27, 29], + [25, 27, 29, 31], + ], + "i2": [ + -15, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + ], + "ai2": [ + [-14, -13, -12], + [-13, -12, -11], + [-12, -11, -10], + [-11, -10, -9], + [-10, -9, -8], + [-9, -8, -7], + [-8, -7, -6], + [-7, -6, -5], + [-6, -5, -4], + [-5, -4, -3], + [-4, -3, -2], + [-3, -2, -1], + [-2, -1, 0], + [-1, 0, 1], + [0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + ], + "Ai2": [ + [], + [-15], + [-15, -13], + [-15, -13, -11], + [-15, -13, -11, -9], + [], + [-10], + [-10, -8], + [-10, -8, -6], + [-10, -8, -6, -4], + [], + [-5], + [-5, -3], + [-5, -3, -1], + [-5, -3, -1, 1], + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + ], + "u2": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + ], + "au2": [ + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + [16, 17, 18], + [17, 18, 19], + [18, 19, 20], + [19, 20, 21], + [20, 21, 22], + [21, 22, 23], + [22, 23, 24], + [23, 24, 25], + [24, 25, 26], + [25, 26, 27], + [26, 27, 28], + [27, 28, 29], + [28, 29, 30], + [29, 30, 31], + [30, 31, 32], + ], + "Au2": [ + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + [], + [15], + [15, 17], + [15, 17, 19], + [15, 17, 19, 21], + [], + [20], + [20, 22], + [20, 22, 24], + [20, 22, 24, 26], + [], + [25], + [25, 27], + [25, 27, 29], + [25, 27, 29, 31], + ], + "i4": [ + -15, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + ], + "ai4": [ + [-14, -13, -12], + [-13, -12, -11], + [-12, -11, -10], + [-11, -10, -9], + [-10, -9, -8], + [-9, -8, -7], + [-8, -7, -6], + [-7, -6, -5], + [-6, -5, -4], + [-5, -4, -3], + [-4, -3, -2], + [-3, -2, -1], + [-2, -1, 0], + [-1, 0, 1], + [0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + ], + "Ai4": [ + [], + [-15], + [-15, -13], + [-15, -13, -11], + [-15, -13, -11, -9], + [], + [-10], + [-10, -8], + [-10, -8, -6], + [-10, -8, -6, -4], + [], + [-5], + [-5, -3], + [-5, -3, -1], + [-5, -3, -1, 1], + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + ], + "u4": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + ], + "au4": [ + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + [16, 17, 18], + [17, 18, 19], + [18, 19, 20], + [19, 20, 21], + [20, 21, 22], + [21, 22, 23], + [22, 23, 24], + [23, 24, 25], + [24, 25, 26], + [25, 26, 27], + [26, 27, 28], + [27, 28, 29], + [28, 29, 30], + [29, 30, 31], + [30, 31, 32], + ], + "Au4": [ + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + [], + [15], + [15, 17], + [15, 17, 19], + [15, 17, 19, 21], + [], + [20], + [20, 22], + [20, 22, 24], + [20, 22, 24, 26], + [], + [25], + [25, 27], + [25, 27, 29], + [25, 27, 29, 31], + ], + "i8": [ + -15, + -14, + -13, + -12, + -11, + -10, + -9, + -8, + -7, + -6, + -5, + -4, + -3, + -2, + -1, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + ], + "ai8": [ + [-14, -13, -12], + [-13, -12, -11], + [-12, -11, -10], + [-11, -10, -9], + [-10, -9, -8], + [-9, -8, -7], + [-8, -7, -6], + [-7, -6, -5], + [-6, -5, -4], + [-5, -4, -3], + [-4, -3, -2], + [-3, -2, -1], + [-2, -1, 0], + [-1, 0, 1], + [0, 1, 2], + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + ], + "Ai8": [ + [], + [-15], + [-15, -13], + [-15, -13, -11], + [-15, -13, -11, -9], + [], + [-10], + [-10, -8], + [-10, -8, -6], + [-10, -8, -6, -4], + [], + [-5], + [-5, -3], + [-5, -3, -1], + [-5, -3, -1, 1], + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + ], + "u8": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + ], + "au8": [ + [1, 2, 3], + [2, 3, 4], + [3, 4, 5], + [4, 5, 6], + [5, 6, 7], + [6, 7, 8], + [7, 8, 9], + [8, 9, 10], + [9, 10, 11], + [10, 11, 12], + [11, 12, 13], + [12, 13, 14], + [13, 14, 15], + [14, 15, 16], + [15, 16, 17], + [16, 17, 18], + [17, 18, 19], + [18, 19, 20], + [19, 20, 21], + [20, 21, 22], + [21, 22, 23], + [22, 23, 24], + [23, 24, 25], + [24, 25, 26], + [25, 26, 27], + [26, 27, 28], + [27, 28, 29], + [28, 29, 30], + [29, 30, 31], + [30, 31, 32], + ], + "Au8": [ + [], + [0], + [0, 2], + [0, 2, 4], + [0, 2, 4, 6], + [], + [5], + [5, 7], + [5, 7, 9], + [5, 7, 9, 11], + [], + [10], + [10, 12], + [10, 12, 14], + [10, 12, 14, 16], + [], + [15], + [15, 17], + [15, 17, 19], + [15, 17, 19, 21], + [], + [20], + [20, 22], + [20, 22, 24], + [20, 22, 24, 26], + [], + [25], + [25, 27], + [25, 27, 29], + [25, 27, 29, 31], + ], + "f4": [ + -14.899999618530273, + -13.899999618530273, + -12.899999618530273, + -11.899999618530273, + -10.899999618530273, + -9.899999618530273, + -8.899999618530273, + -7.900000095367432, + -6.900000095367432, + -5.900000095367432, + -4.900000095367432, + -3.9000000953674316, + -2.9000000953674316, + -1.899999976158142, + -0.8999999761581421, + 0.10000000149011612, + 1.100000023841858, + 2.0999999046325684, + 3.0999999046325684, + 4.099999904632568, + 5.099999904632568, + 6.099999904632568, + 7.099999904632568, + 8.100000381469727, + 9.100000381469727, + 10.100000381469727, + 11.100000381469727, + 12.100000381469727, + 13.100000381469727, + 14.100000381469727, + ], + "af4": [ + [-13.899999618530273, -12.899999618530273, -11.899999618530273], + [-12.899999618530273, -11.899999618530273, -10.899999618530273], + [-11.899999618530273, -10.899999618530273, -9.899999618530273], + [-10.899999618530273, -9.899999618530273, -8.899999618530273], + [-9.899999618530273, -8.899999618530273, -7.900000095367432], + [-8.899999618530273, -7.900000095367432, -6.900000095367432], + [-7.900000095367432, -6.900000095367432, -5.900000095367432], + [-6.900000095367432, -5.900000095367432, -4.900000095367432], + [-5.900000095367432, -4.900000095367432, -3.9000000953674316], + [-4.900000095367432, -3.9000000953674316, -2.9000000953674316], + [-3.9000000953674316, -2.9000000953674316, -1.899999976158142], + [-2.9000000953674316, -1.899999976158142, -0.8999999761581421], + [-1.899999976158142, -0.8999999761581421, 0.10000000149011612], + [-0.8999999761581421, 0.10000000149011612, 1.100000023841858], + [0.10000000149011612, 1.100000023841858, 2.0999999046325684], + [1.100000023841858, 2.0999999046325684, 3.0999999046325684], + [2.0999999046325684, 3.0999999046325684, 4.099999904632568], + [3.0999999046325684, 4.099999904632568, 5.099999904632568], + [4.099999904632568, 5.099999904632568, 6.099999904632568], + [5.099999904632568, 6.099999904632568, 7.099999904632568], + [6.099999904632568, 7.099999904632568, 8.100000381469727], + [7.099999904632568, 8.100000381469727, 9.100000381469727], + [8.100000381469727, 9.100000381469727, 10.100000381469727], + [9.100000381469727, 10.100000381469727, 11.100000381469727], + [10.100000381469727, 11.100000381469727, 12.100000381469727], + [11.100000381469727, 12.100000381469727, 13.100000381469727], + [12.100000381469727, 13.100000381469727, 14.100000381469727], + [13.100000381469727, 14.100000381469727, 15.100000381469727], + [14.100000381469727, 15.100000381469727, 16.100000381469727], + [15.100000381469727, 16.100000381469727, 17.100000381469727], + ], + "Af4": [ + [], + [-15.0], + [-15.0, -13.899999618530273], + [-15.0, -13.899999618530273, -12.800000190734863], + [-15.0, -13.899999618530273, -12.800000190734863, -11.699999809265137], + [], + [-10.0], + [-10.0, -8.899999618530273], + [-10.0, -8.899999618530273, -7.800000190734863], + [-10.0, -8.899999618530273, -7.800000190734863, -6.699999809265137], + [], + [-5.0], + [-5.0, -3.9000000953674316], + [-5.0, -3.9000000953674316, -2.799999952316284], + [-5.0, -3.9000000953674316, -2.799999952316284, -1.7000000476837158], + [], + [0.0], + [0.0, 1.100000023841858], + [0.0, 1.100000023841858, 2.200000047683716], + [0.0, 1.100000023841858, 2.200000047683716, 3.299999952316284], + [], + [5.0], + [5.0, 6.099999904632568], + [5.0, 6.099999904632568, 7.199999809265137], + [5.0, 6.099999904632568, 7.199999809265137, 8.300000190734863], + [], + [10.0], + [10.0, 11.100000381469727], + [10.0, 11.100000381469727, 12.199999809265137], + [10.0, 11.100000381469727, 12.199999809265137, 13.300000190734863], + ], + "f8": [ + -14.9, + -13.9, + -12.9, + -11.9, + -10.9, + -9.9, + -8.9, + -7.9, + -6.9, + -5.9, + -4.9, + -3.9000000000000004, + -2.9000000000000004, + -1.9000000000000004, + -0.9000000000000004, + 0.09999999999999964, + 1.0999999999999996, + 2.0999999999999996, + 3.0999999999999996, + 4.1, + 5.1, + 6.1, + 7.1, + 8.1, + 9.1, + 10.1, + 11.1, + 12.1, + 13.1, + 14.1, + ], + "af8": [ + [-13.9, -12.9, -11.9], + [-12.9, -11.9, -10.9], + [-11.9, -10.9, -9.9], + [-10.9, -9.9, -8.9], + [-9.9, -8.9, -7.9], + [-8.9, -7.9, -6.9], + [-7.9, -6.9, -5.9], + [-6.9, -5.9, -4.9], + [-5.9, -4.9, -3.9000000000000004], + [-4.9, -3.9000000000000004, -2.9000000000000004], + [-3.9000000000000004, -2.9000000000000004, -1.9000000000000004], + [-2.9000000000000004, -1.9000000000000004, -0.9000000000000004], + [-1.9000000000000004, -0.9000000000000004, 0.09999999999999964], + [-0.9000000000000004, 0.09999999999999964, 1.0999999999999996], + [0.09999999999999964, 1.0999999999999996, 2.0999999999999996], + [1.0999999999999996, 2.0999999999999996, 3.0999999999999996], + [2.0999999999999996, 3.0999999999999996, 4.1], + [3.0999999999999996, 4.1, 5.1], + [4.1, 5.1, 6.1], + [5.1, 6.1, 7.1], + [6.1, 7.1, 8.1], + [7.1, 8.1, 9.1], + [8.1, 9.1, 10.1], + [9.1, 10.1, 11.1], + [10.1, 11.1, 12.1], + [11.1, 12.1, 13.1], + [12.1, 13.1, 14.1], + [13.1, 14.1, 15.1], + [14.1, 15.1, 16.1], + [15.1, 16.1, 17.1], + ], + "Af8": [ + [], + [-15.0], + [-15.0, -13.9], + [-15.0, -13.9, -12.8], + [-15.0, -13.9, -12.8, -11.7], + [], + [-10.0], + [-10.0, -8.9], + [-10.0, -8.9, -7.8], + [-10.0, -8.9, -7.8, -6.7], + [], + [-5.0], + [-5.0, -3.9], + [-5.0, -3.9, -2.8], + [-5.0, -3.9, -2.8, -1.7], + [], + [0.0], + [0.0, 1.1], + [0.0, 1.1, 2.2], + [0.0, 1.1, 2.2, 3.3], + [], + [5.0], + [5.0, 6.1], + [5.0, 6.1, 7.2], + [5.0, 6.1, 7.2, 8.3], + [], + [10.0], + [10.0, 11.1], + [10.0, 11.1, 12.2], + [10.0, 11.1, 12.2, 13.3], + ], + "str": [ + "hey-0", + "hey-1", + "hey-2", + "hey-3", + "hey-4", + "hey-5", + "hey-6", + "hey-7", + "hey-8", + "hey-9", + "hey-10", + "hey-11", + "hey-12", + "hey-13", + "hey-14", + "hey-15", + "hey-16", + "hey-17", + "hey-18", + "hey-19", + "hey-20", + "hey-21", + "hey-22", + "hey-23", + "hey-24", + "hey-25", + "hey-26", + "hey-27", + "hey-28", + "hey-29", + ], + } + + import uproot + + with uproot.open("uproot-sample-6.20.04-uncompressed.root")["sample"] as sample: + arrays = sample.arrays(sample.keys(), library="np") + + assert set(arrays.keys()) == set(truth.keys()) + for key in truth.keys(): + if isinstance( + sample[key].interpretation, uproot.interpretation.jagged.AsJagged + ): + assert [row.tolist() for row in arrays[key]] == truth[key] + else: + assert arrays[key].tolist() == truth[key] + + assert sample.file._streamers is None From daed905064f11affa7ce38f55ce02dba85fd7a4f Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Tue, 27 Aug 2024 10:43:25 -0400 Subject: [PATCH 14/20] Fixed tests --- tests-wasm/test_1272_basic_rntuple_reading.py | 4 +++- tests-wasm/utils.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests-wasm/test_1272_basic_rntuple_reading.py b/tests-wasm/test_1272_basic_rntuple_reading.py index cb86228ea..9142101a6 100644 --- a/tests-wasm/test_1272_basic_rntuple_reading.py +++ b/tests-wasm/test_1272_basic_rntuple_reading.py @@ -4,6 +4,7 @@ from utils import run_test_in_pyodide +# Taken from test_1191_rntuple_fixes.py @run_test_in_pyodide(test_file="test_ntuple_extension_columns.root") def test_schema_extension(selenium): import uproot @@ -28,8 +29,9 @@ def test_schema_extension(selenium): assert len(jets.pt) == len(pbs) +# Taken from test_1191_rntuple_fixes.py @run_test_in_pyodide(test_file="Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root") -def test_split_encoding(): +def test_split_encoding(selenium): import uproot with uproot.open("Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root") as f: diff --git a/tests-wasm/utils.py b/tests-wasm/utils.py index 4fb6bce4c..98c00d0b4 100644 --- a/tests-wasm/utils.py +++ b/tests-wasm/utils.py @@ -24,7 +24,7 @@ def ensure_testdata(filename): shutil.copyfile(filepath, "skhep_testdata/" + filename) -def run_test_in_pyodide(test_file=None): +def run_test_in_pyodide(test_file=None, **kwargs): def decorator(test_func): @wraps(test_func) def wrapper(selenium): @@ -41,7 +41,7 @@ def wrapper(selenium): install_wheels=True, ) def inner_func(selenium): - run_in_pyodide()(test_func)(selenium) + run_in_pyodide(**kwargs)(test_func)(selenium) return inner_func(selenium) From 3ad893565dd2ca329d568be802ee937a872989ae Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 28 Aug 2024 09:22:53 -0400 Subject: [PATCH 15/20] Fixed typo --- tests/test_1191_rntuple_fixes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_1191_rntuple_fixes.py b/tests/test_1191_rntuple_fixes.py index 7b64e45d2..a1e259310 100644 --- a/tests/test_1191_rntuple_fixes.py +++ b/tests/test_1191_rntuple_fixes.py @@ -41,7 +41,7 @@ def test_rntuple_cardinality(): def test_skip_recursively_empty_structs(): filename = skhep_testdata.data_path("DAOD_TRUTH3_RC2.root") with uproot.open(filename) as f: - obj = uproot.open(filename)["RNT:CollectionTree"] + obj = f["RNT:CollectionTree"] arrays = obj.arrays() jets = arrays["AntiKt4TruthDressedWZJetsAux:"] assert len(jets[0].pt) == 5 From 5e25812de6358dc14feb218fa40ab4fd3bc3487c Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 28 Aug 2024 11:35:14 -0400 Subject: [PATCH 16/20] Small patches to get things to work on pyodide --- src/uproot/source/fsspec.py | 5 +---- src/uproot/source/http.py | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/uproot/source/fsspec.py b/src/uproot/source/fsspec.py index 0ba1bc61e..f222d8e84 100644 --- a/src/uproot/source/fsspec.py +++ b/src/uproot/source/fsspec.py @@ -169,12 +169,9 @@ def submit(request_ranges: list[tuple[int, int]]): # Threads can't be spawned in pyodide yet, so we run the function directly # and return a future that is already resolved. # TODO: remove this when pyodide supports threads - loop = asyncio.get_event_loop() - future = loop.create_future() - future.set_result( + return uproot.source.futures.TrivialFuture( self._fs.cat_ranges(paths=paths, starts=starts, ends=ends) ) - return future return self._executor.submit(coroutine) return coalesce_requests( diff --git a/src/uproot/source/http.py b/src/uproot/source/http.py index 2f0b64a03..3dcfb14d1 100644 --- a/src/uproot/source/http.py +++ b/src/uproot/source/http.py @@ -239,6 +239,23 @@ def future(source: uproot.source.chunk.Source, start: int, stop: int): Returns a :doc:`uproot.source.futures.ResourceFuture` that calls :ref:`uproot.source.http.HTTPResource.get` with ``start`` and ``stop``. """ + # The default implementation doesn't work in Pyodide + if sys.platform == "emscripten": + + def task(resource): + import requests + + r = requests.get( + source._file_path, + headers=dict( + {"Range": f"bytes={start}-{stop - 1}"}, **source.auth_headers + ), + timeout=source.timeout, + ) + return r.content + + return uproot.source.futures.ResourceFuture(task) + connection = make_connection(source.parsed_url, source.timeout) connection.request( "GET", @@ -281,6 +298,14 @@ def multifuture( ``results`` and ``futures``. Subsequent attempts would immediately use the :ref:`uproot.source.http.HTTPSource.fallback`. """ + # The default implementation doesn't work in Pyodide + if sys.platform == "emscripten": + + def task(resource): + resource.handle_no_multipart(source, ranges, futures, results) + + return uproot.source.futures.ResourceFuture(task) + connection = make_connection(source.parsed_url, source.timeout) connection.request( From 5c857b85532311e37118cd39c5b4b05ca6a3674c Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 28 Aug 2024 11:43:28 -0400 Subject: [PATCH 17/20] Added http tests --- .github/workflows/build-test.yml | 2 +- tests-wasm/test_1272_http_reading.py | 35 ++++++++++++++++++++++++++++ tests-wasm/utils.py | 13 +++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 tests-wasm/test_1272_http_reading.py diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 7f1c664e1..c847e607d 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -178,4 +178,4 @@ jobs: - name: Run pytest run: | - pytest -vv --dist-dir=./pyodide-dist/ --runner=selenium --rt=chrome tests-wasm + pytest -vv --dist-dir=./pyodide-dist/ --runner=selenium --runtime=chrome tests-wasm diff --git a/tests-wasm/test_1272_http_reading.py b/tests-wasm/test_1272_http_reading.py new file mode 100644 index 000000000..915119dbd --- /dev/null +++ b/tests-wasm/test_1272_http_reading.py @@ -0,0 +1,35 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +import pytest +from utils import run_test_in_pyodide + + +# Taken from test_0088_read_with_http.py +@pytest.mark.network() +@run_test_in_pyodide(packages=["requests"]) +def test_ttree(selenium): + import uproot + + with uproot.open( + "http://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root", + handler=uproot.source.http.HTTPSource, + ) as f: + data = f["tree"].arrays(["Y1", "Y2"], library="np") + assert len(data["Y1"]) == 100000 + assert len(data["Y2"]) == 100000 + + +# Taken from test_1191_rntuple_fixes.py +@pytest.mark.network() +@run_test_in_pyodide(packages=["requests"]) +def test_rntuple(selenium): + import uproot + + with uproot.open( + "https://github.com/scikit-hep/scikit-hep-testdata/raw/main/src/skhep_testdata/data/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root", + handler=uproot.source.http.HTTPSource, + ) as f: + obj = f["Events"] + arrays = obj.arrays() + assert arrays["nMuon"].tolist() == [len(a) for a in arrays["Muon_pt"]] diff --git a/tests-wasm/utils.py b/tests-wasm/utils.py index 98c00d0b4..a703b13e2 100644 --- a/tests-wasm/utils.py +++ b/tests-wasm/utils.py @@ -10,11 +10,24 @@ import skhep_testdata try: + import pytest_pyodide from pytest_pyodide import run_in_pyodide from pytest_pyodide.decorator import copy_files_to_pyodide except ImportError: pytest.skip("Pyodide is not available", allow_module_level=True) +# Disable CORS so that we can fetch files for http tests +# Currently, this can only be done for Chrome +selenium_config = pytest_pyodide.config.get_global_config() +selenium_config.set_flags( + "chrome", + [ + *selenium_config.get_flags("chrome"), + "--disable-web-security", + "--disable-site-isolation-trials", + ], +) + # copy skhep_testdata files to testdata directory (needed for @copy_files_to_pyodide) def ensure_testdata(filename): From 357ff8aa3f5226330a39c3d5119a7d55824b03c6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:45:13 +0000 Subject: [PATCH 18/20] style: pre-commit fixes --- tests-wasm/test_1272_http_reading.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests-wasm/test_1272_http_reading.py b/tests-wasm/test_1272_http_reading.py index 915119dbd..06ae60bad 100644 --- a/tests-wasm/test_1272_http_reading.py +++ b/tests-wasm/test_1272_http_reading.py @@ -6,7 +6,7 @@ # Taken from test_0088_read_with_http.py -@pytest.mark.network() +@pytest.mark.network @run_test_in_pyodide(packages=["requests"]) def test_ttree(selenium): import uproot @@ -21,7 +21,7 @@ def test_ttree(selenium): # Taken from test_1191_rntuple_fixes.py -@pytest.mark.network() +@pytest.mark.network @run_test_in_pyodide(packages=["requests"]) def test_rntuple(selenium): import uproot From bdd86ea4239c9fc50943f41004abe4dd70bdbb31 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 28 Aug 2024 12:23:18 -0400 Subject: [PATCH 19/20] Cleaned up and added writing test --- tests-wasm/test_1272_basic_functionality.py | 124 ++ tests-wasm/test_1272_basic_rntuple_reading.py | 44 - tests-wasm/test_1272_basic_ttree_reading.py | 1147 ----------------- tests-wasm/test_1272_http_reading.py | 35 - 4 files changed, 124 insertions(+), 1226 deletions(-) create mode 100644 tests-wasm/test_1272_basic_functionality.py delete mode 100644 tests-wasm/test_1272_basic_rntuple_reading.py delete mode 100644 tests-wasm/test_1272_basic_ttree_reading.py delete mode 100644 tests-wasm/test_1272_http_reading.py diff --git a/tests-wasm/test_1272_basic_functionality.py b/tests-wasm/test_1272_basic_functionality.py new file mode 100644 index 000000000..68d14e575 --- /dev/null +++ b/tests-wasm/test_1272_basic_functionality.py @@ -0,0 +1,124 @@ +# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE +from __future__ import annotations + +import pytest +from utils import run_test_in_pyodide + + +# Taken from test_0034_generic_objects_in_ttrees.py +@run_test_in_pyodide(test_file="uproot-HZZ-objects.root", packages=["pytest", "xxhash"]) +def test_read_ttree(selenium): + import pytest + + import uproot + + awkward = pytest.importorskip("awkward") + + with uproot.open("uproot-HZZ-objects.root")["events"] as tree: + result = tree["muonp4"].array(library="ak") + + assert ( + str(awkward.type(result)) + == "2421 * var * TLorentzVector[fP: TVector3[fX: float64, " + "fY: float64, fZ: float64], fE: float64]" + ) + + assert result[0, 0, "fE"] == 54.77949905395508 + assert result[0, 0, "fP", "fX"] == -52.89945602416992 + assert result[0, 0, "fP", "fY"] == -11.654671669006348 + assert result[0, 0, "fP", "fZ"] == -8.16079330444336 + + +# Taken from test_0406_write_a_tree.py +@run_test_in_pyodide() +def test_write_ttree(selenium): + import numpy as np + + import uproot + + newfile = "newfile.root" + + b1 = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + b2 = [0.0, 1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7, 8.8, 9.9] + + with uproot.recreate(newfile, compression=None) as fout: + tree = fout.mktree("t", {"b1": np.int32, "b2": np.float64}, "title") + + assert tree._cascading._basket_capacity == 10 + + for _ in range(5): + fout["t"].extend({"b1": b1, "b2": b2}) + + assert tree._cascading._basket_capacity == 10 + + for _ in range(10): + fout["t"].extend({"b1": b1, "b2": b2}) + + assert tree._cascading._basket_capacity == 100 + + for _ in range(90): + fout["t"].extend({"b1": b1, "b2": b2}) + + assert tree._cascading._basket_capacity == 1000 + + with uproot.open(newfile) as fin: + assert fin.keys() == ["t;1"] # same cycle number + t2 = fin["t"] + assert t2.num_entries == len(b1) * 105 + assert t2["b1"].array(library="np").tolist() == b1 * 105 + assert t2["b2"].array(library="np").tolist() == b2 * 105 + + +# Taken from test_1191_rntuple_fixes.py +@run_test_in_pyodide(test_file="test_ntuple_extension_columns.root") +def test_read_rntuple(selenium): + import uproot + + with uproot.open("test_ntuple_extension_columns.root") as f: + obj = f["EventData"] + + assert len(obj.column_records) > len(obj.header.column_records) + assert len(obj.column_records) == 936 + assert obj.column_records[903].first_ele_index == 36 + + arrays = obj.arrays() + + pbs = arrays[ + "HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux::fastDIPS20211215_pb" + ] + assert len(pbs) == 40 + assert all(len(a) == 0 for a in pbs[:36]) + assert next(i for i, a in enumerate(pbs) if len(a) != 0) == 36 + + jets = arrays["HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux:"] + assert len(jets.pt) == len(pbs) + + +# Taken from test_0088_read_with_http.py +@pytest.mark.network +@run_test_in_pyodide(packages=["requests"]) +def test_read_ttree_http(selenium): + import uproot + + with uproot.open( + "http://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root", + handler=uproot.source.http.HTTPSource, + ) as f: + data = f["tree"].arrays(["Y1", "Y2"], library="np") + assert len(data["Y1"]) == 100000 + assert len(data["Y2"]) == 100000 + + +# Taken from test_1191_rntuple_fixes.py +@pytest.mark.network +@run_test_in_pyodide(packages=["requests"]) +def test_read_rntuple_http(selenium): + import uproot + + with uproot.open( + "https://github.com/scikit-hep/scikit-hep-testdata/raw/main/src/skhep_testdata/data/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root", + handler=uproot.source.http.HTTPSource, + ) as f: + obj = f["Events"] + arrays = obj.arrays() + assert arrays["nMuon"].tolist() == [len(a) for a in arrays["Muon_pt"]] diff --git a/tests-wasm/test_1272_basic_rntuple_reading.py b/tests-wasm/test_1272_basic_rntuple_reading.py deleted file mode 100644 index 9142101a6..000000000 --- a/tests-wasm/test_1272_basic_rntuple_reading.py +++ /dev/null @@ -1,44 +0,0 @@ -# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE -from __future__ import annotations - -from utils import run_test_in_pyodide - - -# Taken from test_1191_rntuple_fixes.py -@run_test_in_pyodide(test_file="test_ntuple_extension_columns.root") -def test_schema_extension(selenium): - import uproot - - with uproot.open("test_ntuple_extension_columns.root") as f: - obj = f["EventData"] - - assert len(obj.column_records) > len(obj.header.column_records) - assert len(obj.column_records) == 936 - assert obj.column_records[903].first_ele_index == 36 - - arrays = obj.arrays() - - pbs = arrays[ - "HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux::fastDIPS20211215_pb" - ] - assert len(pbs) == 40 - assert all(len(a) == 0 for a in pbs[:36]) - assert next(i for i, a in enumerate(pbs) if len(a) != 0) == 36 - - jets = arrays["HLT_AntiKt4EMPFlowJets_subresjesgscIS_ftf_TLAAux:"] - assert len(jets.pt) == len(pbs) - - -# Taken from test_1191_rntuple_fixes.py -@run_test_in_pyodide(test_file="Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root") -def test_split_encoding(selenium): - import uproot - - with uproot.open("Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root") as f: - obj = f["Events"] - arrays = obj.arrays() - - expected_pt = [10.763696670532227, 15.736522674560547] - expected_charge = [-1, -1] - assert arrays["Muon_pt"][0].tolist() == expected_pt - assert arrays["Muon_charge"][0].tolist() == expected_charge diff --git a/tests-wasm/test_1272_basic_ttree_reading.py b/tests-wasm/test_1272_basic_ttree_reading.py deleted file mode 100644 index 153b75ae9..000000000 --- a/tests-wasm/test_1272_basic_ttree_reading.py +++ /dev/null @@ -1,1147 +0,0 @@ -# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE -from __future__ import annotations - -from utils import run_test_in_pyodide - - -# Taken from test_0023_ttree_versions.py -@run_test_in_pyodide(test_file="uproot-sample-6.20.04-uncompressed.root") -def test_read_ttree(selenium): - truth = { - "n": [ - 0, - 1, - 2, - 3, - 4, - 0, - 1, - 2, - 3, - 4, - 0, - 1, - 2, - 3, - 4, - 0, - 1, - 2, - 3, - 4, - 0, - 1, - 2, - 3, - 4, - 0, - 1, - 2, - 3, - 4, - ], - "b": [ - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - True, - False, - ], - "ab": [ - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - [False, True, False], - [True, False, True], - ], - "Ab": [ - [], - [True], - [True, True], - [True, True, True], - [True, True, True, True], - [], - [False], - [False, False], - [False, False, False], - [False, False, False, False], - [], - [True], - [True, True], - [True, True, True], - [True, True, True, True], - [], - [False], - [False, False], - [False, False, False], - [False, False, False, False], - [], - [True], - [True, True], - [True, True, True], - [True, True, True, True], - [], - [False], - [False, False], - [False, False, False], - [False, False, False, False], - ], - "i1": [ - -15, - -14, - -13, - -12, - -11, - -10, - -9, - -8, - -7, - -6, - -5, - -4, - -3, - -2, - -1, - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - ], - "ai1": [ - [-14, -13, -12], - [-13, -12, -11], - [-12, -11, -10], - [-11, -10, -9], - [-10, -9, -8], - [-9, -8, -7], - [-8, -7, -6], - [-7, -6, -5], - [-6, -5, -4], - [-5, -4, -3], - [-4, -3, -2], - [-3, -2, -1], - [-2, -1, 0], - [-1, 0, 1], - [0, 1, 2], - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - ], - "Ai1": [ - [], - [-15], - [-15, -13], - [-15, -13, -11], - [-15, -13, -11, -9], - [], - [-10], - [-10, -8], - [-10, -8, -6], - [-10, -8, -6, -4], - [], - [-5], - [-5, -3], - [-5, -3, -1], - [-5, -3, -1, 1], - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - ], - "u1": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - ], - "au1": [ - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - [16, 17, 18], - [17, 18, 19], - [18, 19, 20], - [19, 20, 21], - [20, 21, 22], - [21, 22, 23], - [22, 23, 24], - [23, 24, 25], - [24, 25, 26], - [25, 26, 27], - [26, 27, 28], - [27, 28, 29], - [28, 29, 30], - [29, 30, 31], - [30, 31, 32], - ], - "Au1": [ - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - [], - [15], - [15, 17], - [15, 17, 19], - [15, 17, 19, 21], - [], - [20], - [20, 22], - [20, 22, 24], - [20, 22, 24, 26], - [], - [25], - [25, 27], - [25, 27, 29], - [25, 27, 29, 31], - ], - "i2": [ - -15, - -14, - -13, - -12, - -11, - -10, - -9, - -8, - -7, - -6, - -5, - -4, - -3, - -2, - -1, - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - ], - "ai2": [ - [-14, -13, -12], - [-13, -12, -11], - [-12, -11, -10], - [-11, -10, -9], - [-10, -9, -8], - [-9, -8, -7], - [-8, -7, -6], - [-7, -6, -5], - [-6, -5, -4], - [-5, -4, -3], - [-4, -3, -2], - [-3, -2, -1], - [-2, -1, 0], - [-1, 0, 1], - [0, 1, 2], - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - ], - "Ai2": [ - [], - [-15], - [-15, -13], - [-15, -13, -11], - [-15, -13, -11, -9], - [], - [-10], - [-10, -8], - [-10, -8, -6], - [-10, -8, -6, -4], - [], - [-5], - [-5, -3], - [-5, -3, -1], - [-5, -3, -1, 1], - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - ], - "u2": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - ], - "au2": [ - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - [16, 17, 18], - [17, 18, 19], - [18, 19, 20], - [19, 20, 21], - [20, 21, 22], - [21, 22, 23], - [22, 23, 24], - [23, 24, 25], - [24, 25, 26], - [25, 26, 27], - [26, 27, 28], - [27, 28, 29], - [28, 29, 30], - [29, 30, 31], - [30, 31, 32], - ], - "Au2": [ - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - [], - [15], - [15, 17], - [15, 17, 19], - [15, 17, 19, 21], - [], - [20], - [20, 22], - [20, 22, 24], - [20, 22, 24, 26], - [], - [25], - [25, 27], - [25, 27, 29], - [25, 27, 29, 31], - ], - "i4": [ - -15, - -14, - -13, - -12, - -11, - -10, - -9, - -8, - -7, - -6, - -5, - -4, - -3, - -2, - -1, - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - ], - "ai4": [ - [-14, -13, -12], - [-13, -12, -11], - [-12, -11, -10], - [-11, -10, -9], - [-10, -9, -8], - [-9, -8, -7], - [-8, -7, -6], - [-7, -6, -5], - [-6, -5, -4], - [-5, -4, -3], - [-4, -3, -2], - [-3, -2, -1], - [-2, -1, 0], - [-1, 0, 1], - [0, 1, 2], - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - ], - "Ai4": [ - [], - [-15], - [-15, -13], - [-15, -13, -11], - [-15, -13, -11, -9], - [], - [-10], - [-10, -8], - [-10, -8, -6], - [-10, -8, -6, -4], - [], - [-5], - [-5, -3], - [-5, -3, -1], - [-5, -3, -1, 1], - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - ], - "u4": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - ], - "au4": [ - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - [16, 17, 18], - [17, 18, 19], - [18, 19, 20], - [19, 20, 21], - [20, 21, 22], - [21, 22, 23], - [22, 23, 24], - [23, 24, 25], - [24, 25, 26], - [25, 26, 27], - [26, 27, 28], - [27, 28, 29], - [28, 29, 30], - [29, 30, 31], - [30, 31, 32], - ], - "Au4": [ - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - [], - [15], - [15, 17], - [15, 17, 19], - [15, 17, 19, 21], - [], - [20], - [20, 22], - [20, 22, 24], - [20, 22, 24, 26], - [], - [25], - [25, 27], - [25, 27, 29], - [25, 27, 29, 31], - ], - "i8": [ - -15, - -14, - -13, - -12, - -11, - -10, - -9, - -8, - -7, - -6, - -5, - -4, - -3, - -2, - -1, - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - ], - "ai8": [ - [-14, -13, -12], - [-13, -12, -11], - [-12, -11, -10], - [-11, -10, -9], - [-10, -9, -8], - [-9, -8, -7], - [-8, -7, -6], - [-7, -6, -5], - [-6, -5, -4], - [-5, -4, -3], - [-4, -3, -2], - [-3, -2, -1], - [-2, -1, 0], - [-1, 0, 1], - [0, 1, 2], - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - ], - "Ai8": [ - [], - [-15], - [-15, -13], - [-15, -13, -11], - [-15, -13, -11, -9], - [], - [-10], - [-10, -8], - [-10, -8, -6], - [-10, -8, -6, -4], - [], - [-5], - [-5, -3], - [-5, -3, -1], - [-5, -3, -1, 1], - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - ], - "u8": [ - 0, - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 10, - 11, - 12, - 13, - 14, - 15, - 16, - 17, - 18, - 19, - 20, - 21, - 22, - 23, - 24, - 25, - 26, - 27, - 28, - 29, - ], - "au8": [ - [1, 2, 3], - [2, 3, 4], - [3, 4, 5], - [4, 5, 6], - [5, 6, 7], - [6, 7, 8], - [7, 8, 9], - [8, 9, 10], - [9, 10, 11], - [10, 11, 12], - [11, 12, 13], - [12, 13, 14], - [13, 14, 15], - [14, 15, 16], - [15, 16, 17], - [16, 17, 18], - [17, 18, 19], - [18, 19, 20], - [19, 20, 21], - [20, 21, 22], - [21, 22, 23], - [22, 23, 24], - [23, 24, 25], - [24, 25, 26], - [25, 26, 27], - [26, 27, 28], - [27, 28, 29], - [28, 29, 30], - [29, 30, 31], - [30, 31, 32], - ], - "Au8": [ - [], - [0], - [0, 2], - [0, 2, 4], - [0, 2, 4, 6], - [], - [5], - [5, 7], - [5, 7, 9], - [5, 7, 9, 11], - [], - [10], - [10, 12], - [10, 12, 14], - [10, 12, 14, 16], - [], - [15], - [15, 17], - [15, 17, 19], - [15, 17, 19, 21], - [], - [20], - [20, 22], - [20, 22, 24], - [20, 22, 24, 26], - [], - [25], - [25, 27], - [25, 27, 29], - [25, 27, 29, 31], - ], - "f4": [ - -14.899999618530273, - -13.899999618530273, - -12.899999618530273, - -11.899999618530273, - -10.899999618530273, - -9.899999618530273, - -8.899999618530273, - -7.900000095367432, - -6.900000095367432, - -5.900000095367432, - -4.900000095367432, - -3.9000000953674316, - -2.9000000953674316, - -1.899999976158142, - -0.8999999761581421, - 0.10000000149011612, - 1.100000023841858, - 2.0999999046325684, - 3.0999999046325684, - 4.099999904632568, - 5.099999904632568, - 6.099999904632568, - 7.099999904632568, - 8.100000381469727, - 9.100000381469727, - 10.100000381469727, - 11.100000381469727, - 12.100000381469727, - 13.100000381469727, - 14.100000381469727, - ], - "af4": [ - [-13.899999618530273, -12.899999618530273, -11.899999618530273], - [-12.899999618530273, -11.899999618530273, -10.899999618530273], - [-11.899999618530273, -10.899999618530273, -9.899999618530273], - [-10.899999618530273, -9.899999618530273, -8.899999618530273], - [-9.899999618530273, -8.899999618530273, -7.900000095367432], - [-8.899999618530273, -7.900000095367432, -6.900000095367432], - [-7.900000095367432, -6.900000095367432, -5.900000095367432], - [-6.900000095367432, -5.900000095367432, -4.900000095367432], - [-5.900000095367432, -4.900000095367432, -3.9000000953674316], - [-4.900000095367432, -3.9000000953674316, -2.9000000953674316], - [-3.9000000953674316, -2.9000000953674316, -1.899999976158142], - [-2.9000000953674316, -1.899999976158142, -0.8999999761581421], - [-1.899999976158142, -0.8999999761581421, 0.10000000149011612], - [-0.8999999761581421, 0.10000000149011612, 1.100000023841858], - [0.10000000149011612, 1.100000023841858, 2.0999999046325684], - [1.100000023841858, 2.0999999046325684, 3.0999999046325684], - [2.0999999046325684, 3.0999999046325684, 4.099999904632568], - [3.0999999046325684, 4.099999904632568, 5.099999904632568], - [4.099999904632568, 5.099999904632568, 6.099999904632568], - [5.099999904632568, 6.099999904632568, 7.099999904632568], - [6.099999904632568, 7.099999904632568, 8.100000381469727], - [7.099999904632568, 8.100000381469727, 9.100000381469727], - [8.100000381469727, 9.100000381469727, 10.100000381469727], - [9.100000381469727, 10.100000381469727, 11.100000381469727], - [10.100000381469727, 11.100000381469727, 12.100000381469727], - [11.100000381469727, 12.100000381469727, 13.100000381469727], - [12.100000381469727, 13.100000381469727, 14.100000381469727], - [13.100000381469727, 14.100000381469727, 15.100000381469727], - [14.100000381469727, 15.100000381469727, 16.100000381469727], - [15.100000381469727, 16.100000381469727, 17.100000381469727], - ], - "Af4": [ - [], - [-15.0], - [-15.0, -13.899999618530273], - [-15.0, -13.899999618530273, -12.800000190734863], - [-15.0, -13.899999618530273, -12.800000190734863, -11.699999809265137], - [], - [-10.0], - [-10.0, -8.899999618530273], - [-10.0, -8.899999618530273, -7.800000190734863], - [-10.0, -8.899999618530273, -7.800000190734863, -6.699999809265137], - [], - [-5.0], - [-5.0, -3.9000000953674316], - [-5.0, -3.9000000953674316, -2.799999952316284], - [-5.0, -3.9000000953674316, -2.799999952316284, -1.7000000476837158], - [], - [0.0], - [0.0, 1.100000023841858], - [0.0, 1.100000023841858, 2.200000047683716], - [0.0, 1.100000023841858, 2.200000047683716, 3.299999952316284], - [], - [5.0], - [5.0, 6.099999904632568], - [5.0, 6.099999904632568, 7.199999809265137], - [5.0, 6.099999904632568, 7.199999809265137, 8.300000190734863], - [], - [10.0], - [10.0, 11.100000381469727], - [10.0, 11.100000381469727, 12.199999809265137], - [10.0, 11.100000381469727, 12.199999809265137, 13.300000190734863], - ], - "f8": [ - -14.9, - -13.9, - -12.9, - -11.9, - -10.9, - -9.9, - -8.9, - -7.9, - -6.9, - -5.9, - -4.9, - -3.9000000000000004, - -2.9000000000000004, - -1.9000000000000004, - -0.9000000000000004, - 0.09999999999999964, - 1.0999999999999996, - 2.0999999999999996, - 3.0999999999999996, - 4.1, - 5.1, - 6.1, - 7.1, - 8.1, - 9.1, - 10.1, - 11.1, - 12.1, - 13.1, - 14.1, - ], - "af8": [ - [-13.9, -12.9, -11.9], - [-12.9, -11.9, -10.9], - [-11.9, -10.9, -9.9], - [-10.9, -9.9, -8.9], - [-9.9, -8.9, -7.9], - [-8.9, -7.9, -6.9], - [-7.9, -6.9, -5.9], - [-6.9, -5.9, -4.9], - [-5.9, -4.9, -3.9000000000000004], - [-4.9, -3.9000000000000004, -2.9000000000000004], - [-3.9000000000000004, -2.9000000000000004, -1.9000000000000004], - [-2.9000000000000004, -1.9000000000000004, -0.9000000000000004], - [-1.9000000000000004, -0.9000000000000004, 0.09999999999999964], - [-0.9000000000000004, 0.09999999999999964, 1.0999999999999996], - [0.09999999999999964, 1.0999999999999996, 2.0999999999999996], - [1.0999999999999996, 2.0999999999999996, 3.0999999999999996], - [2.0999999999999996, 3.0999999999999996, 4.1], - [3.0999999999999996, 4.1, 5.1], - [4.1, 5.1, 6.1], - [5.1, 6.1, 7.1], - [6.1, 7.1, 8.1], - [7.1, 8.1, 9.1], - [8.1, 9.1, 10.1], - [9.1, 10.1, 11.1], - [10.1, 11.1, 12.1], - [11.1, 12.1, 13.1], - [12.1, 13.1, 14.1], - [13.1, 14.1, 15.1], - [14.1, 15.1, 16.1], - [15.1, 16.1, 17.1], - ], - "Af8": [ - [], - [-15.0], - [-15.0, -13.9], - [-15.0, -13.9, -12.8], - [-15.0, -13.9, -12.8, -11.7], - [], - [-10.0], - [-10.0, -8.9], - [-10.0, -8.9, -7.8], - [-10.0, -8.9, -7.8, -6.7], - [], - [-5.0], - [-5.0, -3.9], - [-5.0, -3.9, -2.8], - [-5.0, -3.9, -2.8, -1.7], - [], - [0.0], - [0.0, 1.1], - [0.0, 1.1, 2.2], - [0.0, 1.1, 2.2, 3.3], - [], - [5.0], - [5.0, 6.1], - [5.0, 6.1, 7.2], - [5.0, 6.1, 7.2, 8.3], - [], - [10.0], - [10.0, 11.1], - [10.0, 11.1, 12.2], - [10.0, 11.1, 12.2, 13.3], - ], - "str": [ - "hey-0", - "hey-1", - "hey-2", - "hey-3", - "hey-4", - "hey-5", - "hey-6", - "hey-7", - "hey-8", - "hey-9", - "hey-10", - "hey-11", - "hey-12", - "hey-13", - "hey-14", - "hey-15", - "hey-16", - "hey-17", - "hey-18", - "hey-19", - "hey-20", - "hey-21", - "hey-22", - "hey-23", - "hey-24", - "hey-25", - "hey-26", - "hey-27", - "hey-28", - "hey-29", - ], - } - - import uproot - - with uproot.open("uproot-sample-6.20.04-uncompressed.root")["sample"] as sample: - arrays = sample.arrays(sample.keys(), library="np") - - assert set(arrays.keys()) == set(truth.keys()) - for key in truth.keys(): - if isinstance( - sample[key].interpretation, uproot.interpretation.jagged.AsJagged - ): - assert [row.tolist() for row in arrays[key]] == truth[key] - else: - assert arrays[key].tolist() == truth[key] - - assert sample.file._streamers is None diff --git a/tests-wasm/test_1272_http_reading.py b/tests-wasm/test_1272_http_reading.py deleted file mode 100644 index 06ae60bad..000000000 --- a/tests-wasm/test_1272_http_reading.py +++ /dev/null @@ -1,35 +0,0 @@ -# BSD 3-Clause License; see https://github.com/scikit-hep/uproot5/blob/main/LICENSE -from __future__ import annotations - -import pytest -from utils import run_test_in_pyodide - - -# Taken from test_0088_read_with_http.py -@pytest.mark.network -@run_test_in_pyodide(packages=["requests"]) -def test_ttree(selenium): - import uproot - - with uproot.open( - "http://starterkit.web.cern.ch/starterkit/data/advanced-python-2019/dalitzdata.root", - handler=uproot.source.http.HTTPSource, - ) as f: - data = f["tree"].arrays(["Y1", "Y2"], library="np") - assert len(data["Y1"]) == 100000 - assert len(data["Y2"]) == 100000 - - -# Taken from test_1191_rntuple_fixes.py -@pytest.mark.network -@run_test_in_pyodide(packages=["requests"]) -def test_rntuple(selenium): - import uproot - - with uproot.open( - "https://github.com/scikit-hep/scikit-hep-testdata/raw/main/src/skhep_testdata/data/Run2012BC_DoubleMuParked_Muons_rntuple_1000evts.root", - handler=uproot.source.http.HTTPSource, - ) as f: - obj = f["Events"] - arrays = obj.arrays() - assert arrays["nMuon"].tolist() == [len(a) for a in arrays["Muon_pt"]] From e9418cdc4e0e477554a80e5e8f1648aa94d06448 Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Thu, 12 Sep 2024 10:13:27 -0400 Subject: [PATCH 20/20] Replaced wasm checks with global boolean --- src/uproot/_util.py | 3 +++ src/uproot/source/coalesce.py | 6 ++---- src/uproot/source/fsspec.py | 4 +--- src/uproot/source/http.py | 4 ++-- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/uproot/_util.py b/src/uproot/_util.py index 11c1d5bc4..96e9e8d21 100644 --- a/src/uproot/_util.py +++ b/src/uproot/_util.py @@ -13,6 +13,7 @@ import numbers import os import re +import sys import warnings from collections.abc import Iterable from pathlib import Path @@ -27,6 +28,8 @@ import uproot.source.fsspec import uproot.source.object +wasm = sys.platform in ("emscripten", "wasi") + def tobytes(array): """ diff --git a/src/uproot/source/coalesce.py b/src/uproot/source/coalesce.py index 824234535..ab3ada9b7 100644 --- a/src/uproot/source/coalesce.py +++ b/src/uproot/source/coalesce.py @@ -6,7 +6,6 @@ from __future__ import annotations import queue -import sys from concurrent.futures import Future from dataclasses import dataclass from typing import Callable @@ -34,7 +33,7 @@ def add_done_callback(self, callback, *, context=None): self._parent.add_done_callback(callback) def result(self, timeout=None): - if sys.platform == "emscripten": + if uproot._util.wasm: # Pyodide futures don't support timeout return self._parent.result()[self._s] return self._parent.result(timeout=timeout)[self._s] @@ -130,9 +129,8 @@ def coalesce_requests( def chunkify(req: RangeRequest): chunk = uproot.source.chunk.Chunk(source, req.start, req.stop, req.future) - if sys.platform == "emscripten": + if uproot._util.wasm: # Callbacks don't work in pyodide yet, so we call the notifier directly - # TODO: Remove this once pyodide supports threads uproot.source.chunk.notifier(chunk, notifications)() else: req.future.add_done_callback( diff --git a/src/uproot/source/fsspec.py b/src/uproot/source/fsspec.py index f222d8e84..6f47890ba 100644 --- a/src/uproot/source/fsspec.py +++ b/src/uproot/source/fsspec.py @@ -5,7 +5,6 @@ import asyncio import concurrent.futures import queue -import sys import fsspec import fsspec.asyn @@ -165,10 +164,9 @@ def submit(request_ranges: list[tuple[int, int]]): self._fs.cat_ranges, paths=paths, starts=starts, ends=ends ) ) - if sys.platform == "emscripten": + if uproot._util.wasm: # Threads can't be spawned in pyodide yet, so we run the function directly # and return a future that is already resolved. - # TODO: remove this when pyodide supports threads return uproot.source.futures.TrivialFuture( self._fs.cat_ranges(paths=paths, starts=starts, ends=ends) ) diff --git a/src/uproot/source/http.py b/src/uproot/source/http.py index 3dcfb14d1..e3fb1e617 100644 --- a/src/uproot/source/http.py +++ b/src/uproot/source/http.py @@ -240,7 +240,7 @@ def future(source: uproot.source.chunk.Source, start: int, stop: int): :ref:`uproot.source.http.HTTPResource.get` with ``start`` and ``stop``. """ # The default implementation doesn't work in Pyodide - if sys.platform == "emscripten": + if uproot._util.wasm: def task(resource): import requests @@ -299,7 +299,7 @@ def multifuture( use the :ref:`uproot.source.http.HTTPSource.fallback`. """ # The default implementation doesn't work in Pyodide - if sys.platform == "emscripten": + if uproot._util.wasm: def task(resource): resource.handle_no_multipart(source, ranges, futures, results)