DOC: Auto-populate related software using mne-installers yaml

.gitignore

@@ -69,10 +69,11 @@ pip-log.txt
 !.coveragerc
 coverage.xml
 tags
-doc/coverages
-doc/samples
-doc/fil-result
-doc/optipng.exe
+/doc/coverages
+/doc/samples
+/doc/fil-result
+/doc/optipng.exe
+/doc/sphinxext/.joblib
 sg_execution_times.rst
 sg_api_usage.rst
 sg_api_unused.dot

doc/conf.py

@@ -112,6 +112,7 @@
     "mne_substitutions",
     "newcontrib_substitutions",
     "unit_role",
+    "related_software",
 ]
 
 # Add any paths that contain templates here, relative to this directory.

doc/install/mne_tools_suite.rst

@@ -30,45 +30,16 @@ Related software
   C++ and is primarily intended for embedded and real-time applications.
 
 There is also a growing ecosystem of other Python packages that work alongside
-MNE-Python, including packages for:
+MNE-Python, including:
 
 .. note:: Something missing?
     :class: sidebar
 
     If you know of a package that is related but not listed here, feel free to
-    :ref:`make a pull request <contributing>` to add it to this list.
-
-- a graphical user interface for MNE-Python (`MNELAB`_)
-- easily importing MEG data from the Human Connectome Project for
-  use with MNE-Python (`MNE-HCP`_)
-- managing MNE projects so that they comply with the `Brain
-  Imaging Data Structure`_ specification (`MNE-BIDS`_)
-- automatic bad channel detection and interpolation (`autoreject`_)
-- convolutional sparse dictionary learning and waveform shape estimation
-  (`alphaCSC`_)
-- independent component analysis (ICA) with good performance on real data
-  (`PICARD`_)
-- automatic labeling of ICA components (`MNE-ICAlabel`_)
-- phase-amplitude coupling (`pactools`_)
-- representational similarity analysis (`rsa`_)
-- microstate analysis (`microstate`_)
-- connectivity analysis using dynamic imaging of coherent sources (DICS)
-  (`conpy`_)
-- other connectivity algorithms (`MNE-Connectivity`_)
-- general-purpose statistical analysis of M/EEG data (`eelbrain`_)
-- post-hoc modification of linear models (`posthoc`_)
-- a python implementation of the Preprocessing Pipeline (PREP) for EEG data
-  (`pyprep`_)
-- automatic multi-dipole localization and uncertainty quantification with
-  the Bayesian algorithm SESAME (`sesameeg`_)
-- GLM and group level analysis of near-infrared spectroscopy data (`MNE-NIRS`_)
-- All-Resolutions Inference (ARI) for statistically valid circular inference
-  and effect localization (`MNE-ARI`_)
-- real-time analysis (`MNE-Realtime`_)
-- non-parametric sequential analyses and adaptive sample size determination (`niseq`_)
-- a graphical user interface for multi-subject MEG/EEG analysis with plugin support (`Meggie`_)
-- A method to localize the superficial generators together with their spatial extent
-  using the Maximum Entropy on the Mean (`MEM`_)
+    to add it to this list by :ref:`making a pull request <contributing>` to update
+    `doc/sphinxext/related_software.py <https://github.com/mne-tools/mne-python/blob/main/doc/sphinxext/related_software.py>`__.
+
+.. related-software::
 
 What should I install?
 ^^^^^^^^^^^^^^^^^^^^^^

doc/sphinxext/related_software.py

@@ -0,0 +1,235 @@
+"""Create a list of related software.
+
+To add a package to the list:
+
+1. Add it to the MNE-installers if possible, and it will automatically appear.
+2. If it's on PyPI and not in the MNE-installers, add it to the PYPI_PACKAGES set.
+3. If it's not on PyPI, add it to the MANUAL_PACKAGES dictionary.
+"""
+
+import functools
+import importlib.metadata
+import os
+import pathlib
+import urllib.request
+
+import joblib
+from docutils import nodes
+from docutils.parsers.rst import Directive
+from sphinx.errors import ExtensionError
+from sphinx.util.display import status_iterator
+
+# If a package is in MNE-Installers, it will be automatically added!
+
+# If it's available on PyPI, add it to this set:
+PYPI_PACKAGES = {
+    "alphaCSC",
+    "conpy",
+    "meggie",
+    "niseq",
+    "sesameeg",
+}
+
+# If it's not available on PyPI, add it to this dict:
+MANUAL_PACKAGES = {
+    # TODO: These packages are not pip-installable as of 2024/07/17, so we have to
+    # manually populate them -- should open issues on their package repos.
+    "best-python": {
+        "Home-page": "https://github.com/multifunkim/best-python",
+        "Summary": "The goal of this project is to provide a way to use the best-brainstorm Matlab solvers in Python, compatible with MNE-Python.",  # noqa: E501
+    },
+    "mne-hcp": {
+        "Home-page": "https://github.com/mne-tools/mne-hcp",
+        "Summary": "We provide Python tools for seamless integration of MEG data from the Human Connectome Project into the Python ecosystem",  # noqa: E501
+    },
+    "posthoc": {
+        "Home-page": "https://users.aalto.fi/~vanvlm1/posthoc/python",
+        "Summary": "post-hoc modification of linear models",
+    },
+    # This package does not provide wheels, so don't force CircleCI to build it.
+    # If it eventually provides binary wheels we could add it to
+    # `tools/circleci_dependencies.sh` and remove from here.
+    "eelbrain": {
+        "Home-page": "https://eelbrain.readthedocs.io/en/stable/",
+        "Summary": "Open-source Python toolkit for MEG and EEG data analysis.",
+    },
+    # mne-kit-gui requires mayavi (ugh)
+    "mne-kit-gui": {
+        "Home-page": "https://github.com/mne-tools/mne-kit-gui",
+        "Summary": "A module for KIT MEG coregistration.",
+    },
+    # fsleyes requires wxpython, which needs to build
+    "fsleyes": {
+        "Home-page": "https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/FSLeyes",
+        "Summary": "FSLeyes is the FSL image viewer.",
+    },
+    # dcm2niix must be built from source
+    "dcm2niix": {
+        "Home-page": "https://github.com/rordenlab/dcm2niix",
+        "Summary": "DICOM to NIfTI converter",
+    },
+    # TODO: mnelab forces PySide6, it can be added to `tools/circleci_dependencies.sh`
+    # when we use PySide6 for doc building. Also its package does not set the Home-page
+    # property.
+    "mnelab": {
+        "Home-page": "https://github.com/cbrnr/mnelab",
+        "Summary": "A graphical user interface for MNE",
+    },
+    # TODO: these do not set a valid homepage or documentation page on PyPI
+    "mne-features": {
+        "Home-page": "https://mne.tools/mne-features",
+        "Summary": "MNE-Features software for extracting features from multivariate time series",  # noqa: E501
+    },
+    "mne-rsa": {
+        "Home-page": "https://users.aalto.fi/~vanvlm1/mne-rsa",
+        "Summary": "Code for performing Representational Similarity Analysis on MNE-Python data structures.",  # noqa: E501
+    },
+    "mffpy": {
+        "Home-page": "https://github.com/BEL-Public/mffpy",
+        "Summary": "Reader and Writer for Philips' MFF file format.",
+    },
+    "emd": {
+        "Home-page": "https://emd.readthedocs.io/en/stable",
+        "Summary": "Empirical Mode Decomposition in Python.",
+    },
+}
+
+REQUIRE_METADATA = os.getenv("MNE_REQUIRE_RELATED_SOFTWARE_INSTALLED", "false").lower()
+REQUIRE_METADATA = REQUIRE_METADATA in ("true", "1")
+
+# These packages pip-install with a different name than the package name
+RENAMES = {
+    "python-neo": "neo",
+    "matplotlib-base": "matplotlib",
+}
+
+_memory = joblib.Memory(location=pathlib.Path(__file__).parent / ".joblib", verbose=0)
+
+
+@_memory.cache(cache_validation_callback=joblib.expires_after(days=7))
+def _get_installer_packages():
+    """Get the MNE-Python installer package list YAML."""
+    with urllib.request.urlopen(
+        "https://raw.githubusercontent.com/mne-tools/mne-installers/main/recipes/mne-python/construct.yaml"
+    ) as url:
+        data = url.read().decode("utf-8")
+    # Parse data for list of names of packages
+    lines = [line.strip() for line in data.splitlines()]
+    start_idx = lines.index("# <<< BEGIN RELATED SOFTWARE LIST >>>") + 1
+    stop_idx = lines.index("# <<< END RELATED SOFTWARE LIST >>>")
+    packages = [
+        # Lines look like
+        # - mne-ari =0.0.0
+        # or similar.
+        line.split()[1]
+        for line in lines[start_idx:stop_idx]
+        if not line.startswith("#")
+    ]
+    return packages
+
+
+@functools.lru_cache
+def _get_packages():
+    packages = _get_installer_packages()
+    # There can be duplicates in manual and installer packages because some of the
+    # PyPI entries for installer packages are incorrect or unusable (see above), so
+    # we don't enforce that. But PyPI and manual should be disjoint:
+    dups = set(MANUAL_PACKAGES) & set(PYPI_PACKAGES)
+    assert not dups, f"Duplicates in MANUAL_PACKAGES and PYPI_PACKAGES: {sorted(dups)}"
+    # And the installer and PyPI-only should be disjoint:
+    dups = set(PYPI_PACKAGES) & set(packages)
+    assert (
+        not dups
+    ), f"Duplicates in PYPI_PACKAGES and installer packages: {sorted(dups)}"
+    for name in PYPI_PACKAGES | set(MANUAL_PACKAGES):
+        if name not in packages:
+            packages.append(name)
+    # Simple alphabetical order
+    packages = sorted(packages, key=lambda x: x.lower())
+    packages = [RENAMES.get(package, package) for package in packages]
+    out = dict()
+    for package in status_iterator(
+        packages, f"Adding {len(packages)} related software packages: "
+    ):
+        out[package] = dict()
+        try:
+            if package in MANUAL_PACKAGES:
+                md = MANUAL_PACKAGES[package]
+            else:
+                md = importlib.metadata.metadata(package)
+        except importlib.metadata.PackageNotFoundError:
+            pass  # raise a complete error later
+        else:
+            # Every project should really have this
+            for key in ("Summary",):
+                if key not in md:
+                    raise ExtensionError(f"Missing {repr(key)} for {package}")
+            # It is annoying to find the home page
+            url = None
+            if "Home-page" in md:
+                url = md["Home-page"]
+            else:
+                for prefix in ("homepage", "documentation"):
+                    for key, val in md.items():
+                        if key == "Project-URL" and val.lower().startswith(
+                            f"{prefix}, "
+                        ):
+                            url = val.split(", ", 1)[1]
+                            break
+                    if url is not None:
+                        break
+                else:
+                    raise RuntimeError(
+                        f"Could not find Home-page for {package} in:\n"
+                        f"{sorted(set(md))}\nwith Summary:\n{md['Summary']}"
+                    )
+            out[package]["url"] = url
+            out[package]["description"] = md["Summary"].replace("\n", "")
+    bad = [package for package in packages if not out[package]]
+    if bad and REQUIRE_METADATA:
+        raise ExtensionError(f"Could not find metadata for:\n{' '.join(bad)}")
+
+    return out
+
+
+class RelatedSoftwareDirective(Directive):
+    """Create a directive that inserts a bullet list of related software."""
+
+    def run(self):
+        """Run the directive."""
+        my_list = nodes.bullet_list(bullet="*")
+        for package, data in _get_packages().items():
+            item = nodes.list_item()
+            if "description" not in data:
+                para = nodes.paragraph(text=f"{package}")
+            else:
+                para = nodes.paragraph(text=f": {data['description']}")
+                refnode = nodes.reference(
+                    "url",
+                    package,
+                    internal=False,
+                    refuri=data["url"],
+                )
+                para.insert(0, refnode)
+            item += para
+            my_list.append(item)
+        return [my_list]
+
+
+def setup(app):
+    app.add_directive("related-software", RelatedSoftwareDirective)
+    # Run it as soon as this is added as a Sphinx extension so that any errors
+    # / new packages are reported early. The next call in run() will be cached.
+    _get_packages()
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }
+
+
+if __name__ == "__main__":  # pragma: no cover
+    items = list(RelatedSoftwareDirective.run(None)[0].children)
+    print(f"Got {len(items)} related software packages:")
+    for item in items:
+        print(f"- {item.astext()}")

pyproject.toml

@@ -307,6 +307,7 @@ ignore_directives = [
     "minigallery",
     "tabularcolumns",
     "toctree",
+    "related-software",
     "rst-class",
     "tab-set",
     "towncrier-draft-entries",

tools/circleci_bash_env.sh

@@ -17,6 +17,7 @@ echo "export MNE_3D_BACKEND=pyvistaqt" >> $BASH_ENV
 echo "export MNE_BROWSER_BACKEND=qt" >> $BASH_ENV
 echo "export MNE_BROWSER_PRECOMPUTE=false" >> $BASH_ENV
 echo "export MNE_ADD_CONTRIBUTOR_IMAGE=true" >> $BASH_ENV
+echo "export MNE_REQUIRE_RELATED_SOFTWARE_INSTALLED=true" >> $BASH_ENV
 echo "export PATH=~/.local/bin/:$PATH" >> $BASH_ENV
 echo "export DISPLAY=:99" >> $BASH_ENV
 echo "source ~/python_env/bin/activate" >> $BASH_ENV

tools/circleci_dependencies.sh

@@ -4,6 +4,13 @@ python -m pip install --upgrade "pip!=20.3.0" build
 # https://github.com/dipy/dipy/issues/3265 for numpy, dipy
 python -m pip install --upgrade --progress-bar off \
     --only-binary "numpy,dipy,scipy,matplotlib,pandas,statsmodels" \
-    -ve .[full,test,doc] "numpy<2" "dipy!=1.9.0" \
+    -ve .[full,test,doc] "numpy==1.26.4" "dipy!=1.9.0" \
     "git+https://github.com/larsoner/pyvista.git@refcycle" \
-    git+https://github.com/sphinx-gallery/sphinx-gallery.git
+    git+https://github.com/sphinx-gallery/sphinx-gallery.git \
+    \
+    alphaCSC autoreject bycycle conpy emd fooof meggie \
+    mne-ari mne-bids-pipeline mne-faster mne-features \
+    mne-icalabel mne-lsl mne-microstates mne-nirs mne-rsa \
+    neurodsp neurokit2 niseq nitime openneuro-py pactools \
+    plotly pycrostates pyprep pyriemann python-picard sesameeg \
+    sleepecg tensorpac yasa