Merge branch 'main' into cuml_enabled

.cruft.json

@@ -1,7 +1,7 @@
 {
   "template": "https://github.com/scverse/cookiecutter-scverse",
-  "commit": "8e96abb5c3e2d5078c44713958da672711cf2a48",
-  "checkout": null,
+  "commit": "87a407a65408d75a949c0b54b19fd287475a56f8",
+  "checkout": "v0.4.0",
   "context": {
     "cookiecutter": {
       "project_name": "PopV",
@@ -13,7 +13,8 @@
       "project_repo": "https://github.com/YosefLab/PopV.git",
       "license": "MIT License",
       "_copy_without_render": [
-        ".github/workflows/**.yaml",
+        ".github/workflows/build.yaml",
+        ".github/workflows/test.yaml",
         "docs/_templates/autosummary/**.rst"
       ],
       "_render_devdocs": false,

.github/workflows/build.yml

@@ -2,9 +2,13 @@ name: Check Build
 
 on:
   push:
-    branches: [main]
+    branches: [main, "[0-9]+.[0-9]+.x"]
   pull_request:
-    branches: [main]
+    branches: [main, "[0-9]+.[0-9]+.x"]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   package:

.github/workflows/test_linux_cuda.yml

@@ -0,0 +1,70 @@
+name: PopV (cuda)
+
+on:
+  push:
+    branches: [main, "[0-9]+.[0-9]+.x"] #this is new
+  pull_request:
+    branches: [main, "[0-9]+.[0-9]+.x"]
+    types: [labeled, synchronize, opened]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    # if PR has label "cuda tests" or "all tests" or if scheduled or manually triggered
+    if: >-
+      (
+        contains(github.event.pull_request.labels.*.name, 'cuda tests') ||
+        contains(github.event.pull_request.labels.*.name, 'all tests') ||
+        contains(github.event_name, 'schedule') ||
+        contains(github.event_name, 'workflow_dispatch')
+      )
+
+    runs-on: [self-hosted, Linux, X64, CUDA]
+
+    defaults:
+      run:
+        shell: bash -e {0} # -e to fail on error
+
+    container:
+      image: ghcr.io/yoseflab/popv:py3.10-cu12-base
+      options: --user root --gpus all --pull always
+
+    name: integration
+
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
+          cache: "pip"
+          cache-dependency-path: "**/pyproject.toml"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip wheel uv
+          python -m uv pip install --system "PopV[tests] @ ."
+          python -m pip install jax[cuda]
+          python -m pip install nvidia-nccl-cu12
+
+      - name: Run pytest
+        env:
+          MPLBACKEND: agg
+          PLATFORM: ${{ matrix.os }}
+          DISPLAY: :42
+          COLUMNS: 120
+        run: |
+          coverage run -m pytest -v --color=yes --accelerator cuda --devices auto
+          coverage report
+
+      - uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -20,6 +20,7 @@ __pycache__/
 /.pytest_cache/
 /.cache/
 /data/
+/node_modules/
 
 # docs
 /docs/generated/

.pre-commit-config.yaml

@@ -2,20 +2,20 @@ fail_fast: false
 default_language_version:
   python: python3
 default_stages:
-  - commit
-  - push
+  - pre-commit
+  - pre-push
 minimum_pre_commit_version: 2.16.0
 repos:
   - repo: https://github.com/psf/black
-    rev: "23.11.0"
+    rev: "24.10.0"
     hooks:
       - id: black
   - repo: https://github.com/asottile/blacken-docs
-    rev: 1.16.0
+    rev: 1.19.1
     hooks:
       - id: blacken-docs
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.1.0
+    rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
         # Newer versions of node don't work on systems that have an older version of GLIBC
@@ -25,12 +25,12 @@ repos:
         # https://github.com/jupyterlab/jupyterlab/issues/12675
         language_version: "17.9.1"
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.7
+    rev: v0.8.2
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: detect-private-key
       - id: check-ast

.pre-commit-config.yaml.rej

@@ -0,0 +1,34 @@
+diff a/.pre-commit-config.yaml b/.pre-commit-config.yaml	(rejected hunks)
+@@ -6,29 +6,18 @@ default_stages:
+   - push
+ minimum_pre_commit_version: 2.16.0
+ repos:
+-  - repo: https://github.com/psf/black
+-    rev: "24.4.2"
+-    hooks:
+-      - id: black
+-  - repo: https://github.com/asottile/blacken-docs
+-    rev: 1.16.0
+-    hooks:
+-      - id: blacken-docs
+   - repo: https://github.com/pre-commit/mirrors-prettier
+     rev: v4.0.0-alpha.8
+     hooks:
+       - id: prettier
+-        # Newer versions of node don't work on systems that have an older version of GLIBC
+-        # (in particular Ubuntu 18.04 and Centos 7)
+-        # EOL of Centos 7 is in 2024-06, we can probably get rid of this then.
+-        # See https://github.com/scverse/cookiecutter-scverse/issues/143 and
+-        # https://github.com/jupyterlab/jupyterlab/issues/12675
+-        language_version: "17.9.1"
+   - repo: https://github.com/astral-sh/ruff-pre-commit
+     rev: v0.4.4
+     hooks:
+       - id: ruff
++        types_or: [python, pyi, jupyter]
+         args: [--fix, --exit-non-zero-on-fix]
++      - id: ruff-format
++        types_or: [python, pyi, jupyter]
+   - repo: https://github.com/pre-commit/pre-commit-hooks
+     rev: v4.6.0
+     hooks:

.readthedocs.yaml

@@ -1,16 +1,17 @@
-# https://docs.readthedocs.io/en/stable/config-file/v2.html
 version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.10"
+    python: "3.11"
 sphinx:
   configuration: docs/conf.py
-  # disable this for more lenient docs builds
-  fail_on_warning: true
 python:
   install:
     - method: pip
       path: .
       extra_requirements:
-        - doc
+        - docsbuild
+submodules:
+  include:
+    - "docs/notebooks"
+  recursive: true

README.md.rej

@@ -0,0 +1,10 @@
+diff a/README.md b/README.md	(rejected hunks)
+@@ -17,7 +17,7 @@ Please refer to the [documentation][link-docs]. In particular, the
+
+ ## Installation
+
+-You need to have Python 3.9 or newer installed on your system. If you don't have
++You need to have Python 3.10 or newer installed on your system. If you don't have
+ Python installed, we recommend installing [Mambaforge](https://github.com/conda-forge/miniforge#mambaforge).
+
+ There are several alternative options to install PopV:

docs/conf.py

@@ -15,6 +15,15 @@
 from datetime import datetime
 from importlib.metadata import metadata
 from pathlib import Path
+import importlib.util
+import inspect
+import os
+import re
+import subprocess
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from typing import Any
 
 HERE = Path(__file__).parent
 sys.path.insert(0, str(HERE / "extensions"))
@@ -40,10 +49,10 @@
 
 html_context = {
     "display_github": True,  # Integrate GitHub
-    "github_user": "yoseflab",  # Username
-    "github_repo": project_name,  # Repo name
-    "github_version": "main",  # Version
-    "conf_py_path": "/docs/",  # Path in the checkout to the docs root
+    "github_user": "cane11",
+    "github_repo": "https://github.com/YosefLab/PopV.git",
+    "github_version": "main",
+    "conf_py_path": "/docs/",
 }
 
 # -- General configuration ---------------------------------------------------
@@ -62,9 +71,16 @@
     "sphinxcontrib.bibtex",
     "sphinx_autodoc_typehints",
     "sphinx.ext.mathjax",
-    "IPython.sphinxext.ipython_console_highlighting",
-    "sphinxext.opengraph",
+    "sphinx.ext.napoleon",
+    "sphinx_autodoc_typehints",  # needs to be after napoleon
+    "sphinx.ext.extlinks",
+    "sphinx.ext.autosummary",
+    "sphinxcontrib.bibtex",
     *[p.stem for p in (HERE / "extensions").glob("*.py")],
+    "sphinx_copybutton",
+    "sphinx_design",
+    "sphinxext.opengraph",
+    "hoverxref.extension",
 ]
 
 autosummary_generate = True
@@ -196,16 +212,76 @@ def linkcode_resolve(domain, info):
 ]
 
 
-def setup(app):
-    """App setup hook."""
-    app.add_config_value(
-        "recommonmark_config",
-        {
-            "auto_toc_tree_section": "Contents",
-            "enable_auto_toc_tree": True,
-            "enable_math": True,
-            "enable_inline_math": False,
-            "enable_eval_rst": True,
-        },
-        True,
-    )
+# -- Config for linkcode -------------------------------------------
+
+
+def git(*args):
+    """Run git command and return output as string."""
+    return subprocess.check_output(["git", *args]).strip().decode()
+
+
+# https://github.com/DisnakeDev/disnake/blob/7853da70b13fcd2978c39c0b7efa59b34d298186/docs/conf.py#L192
+# Current git reference. Uses branch/tag name if found, otherwise uses commit hash
+git_ref = None
+try:
+    git_ref = git("name-rev", "--name-only", "--no-undefined", "HEAD")
+    git_ref = re.sub(r"^(remotes/[^/]+|tags)/", "", git_ref)
+except Exception:  # noqa: BLE001
+    pass
+
+# (if no name found or relative ref, use commit hash instead)
+if not git_ref or re.search(r"[\^~]", git_ref):
+    try:
+        git_ref = git("rev-parse", "HEAD")
+    except Exception:  # noqa: BLE001
+        git_ref = "main"
+
+# https://github.com/DisnakeDev/disnake/blob/7853da70b13fcd2978c39c0b7efa59b34d298186/docs/conf.py#L192
+_scvi_tools_module_path = os.path.dirname(importlib.util.find_spec("scvi").origin)  # type: ignore
+
+
+def linkcode_resolve(domain, info):
+    """Determine the URL corresponding to Python object."""
+    if domain != "py":
+        return None
+
+    try:
+        obj: Any = sys.modules[info["module"]]
+        for part in info["fullname"].split("."):
+            obj = getattr(obj, part)
+        obj = inspect.unwrap(obj)
+
+        if isinstance(obj, property):
+            obj = inspect.unwrap(obj.fget)  # type: ignore
+
+        path = os.path.relpath(inspect.getsourcefile(obj), start=_scvi_tools_module_path)  # type: ignore
+        src, lineno = inspect.getsourcelines(obj)
+    except Exception:  # noqa: BLE001
+        return None
+
+    path = f"{path}#L{lineno}-L{lineno + len(src) - 1}"
+    return f"{repository_url}/blob/{git_ref}/src/scvi/{path}"
+
+
+# -- Config for hoverxref -------------------------------------------
+
+hoverx_default_type = "tooltip"
+hoverxref_domains = ["py"]
+hoverxref_role_types = dict.fromkeys(
+    ["ref", "class", "func", "meth", "attr", "exc", "data", "mod"],
+    "tooltip",
+)
+hoverxref_intersphinx = [
+    "python",
+    "numpy",
+    "scanpy",
+    "anndata",
+    "pytorch_lightning",
+    "scipy",
+    "pandas",
+    "ml_collections",
+    "ray",
+]
+# use proxied API endpoint on rtd to avoid CORS issues
+if os.environ.get("READTHEDOCS"):
+    hoverxref_api_host = "/_"

docs/contributing.md

@@ -51,7 +51,7 @@ and [prettier][prettier-editors].
 ## Writing tests
 
 ```{note}
-Remember to first install the package with `pip install '-e[dev,test]'`
+Remember to first install the package with `pip install -e '.[dev,test]'`
 ```
 
 This package uses the [pytest][] for automated testing. Please [write tests][scanpy-test-docs] for every function added
@@ -93,7 +93,7 @@ Before making a release, you need to update the version number in the `pyproject
 > Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
 
 Once you are done, commit and push your changes and navigate to the "Releases" page of this project on GitHub.
-Specify `vX.X.X` as a tag name and create a release. For more information, see [managing Github releases][]. This will automatically create a git tag and trigger a Github workflow that creates a release on PyPI.
+Specify `vX.X.X` as a tag name and create a release. For more information, see [managing GitHub releases][]. This will automatically create a git tag and trigger a Github workflow that creates a release on PyPI.
 
 ## Writing documentation
 
@@ -157,3 +157,4 @@ open _build/html/index.html
 [numpydoc]: https://numpydoc.readthedocs.io/en/latest/format.html
 [sphinx autodoc typehints]: https://github.com/tox-dev/sphinx-autodoc-typehints
 [pypi]: https://pypi.org/
+[managing GitHub releases]: https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository