TST: Fix testsuite rattler

.github/workflows/ci.yml

@@ -54,11 +54,11 @@ jobs:
 
       - name: Install dependencies (standard)
         if: matrix.python-version != '3.12.0-rc.2'
-        run: python -m pip install ".[test,hg,testR]"
+        run: python -m pip install ".[test,hg,testR,envs]"
 
       - name: Install dependencies (with --pre)
         if: matrix.python-version == '3.12.0-rc.2'
-        run: python -m pip install ".[test,hg,testR]" --pre
+        run: python -m pip install ".[test,hg,testR,envs]" --pre
 
       - name: Install asv
         run: pip install .
@@ -92,7 +92,7 @@ jobs:
             conda-build
 
       - name: Install dependencies
-        run: python -m pip install ".[test,hg]" --pre
+        run: python -m pip install ".[test,hg,envs]" --pre
         shell: micromamba-shell {0}
 
       - name: Install asv

.github/workflows/ci_win.yml

@@ -31,9 +31,9 @@ jobs:
       - name: Install and test
         shell: pwsh
         run: |
-          python.exe -m pip install .[test]
-          python.exe -m pip install packaging virtualenv
-          python.exe -m pytest -v -l -x --timeout=300 --durations=100 test --environment-type=virtualenv
+          python.exe -m pip install .[test,envs]
+          python.exe -m pip install packaging
+          python.exe -m pytest -v -l -x --timeout=300 --durations=100 test --environment-type=rattler
 
 
   test_env:
@@ -59,7 +59,7 @@ jobs:
             conda-build
 
       - name: Install dependencies
-        run: python -m pip install ".[test,hg]" --pre
+        run: python -m pip install ".[test,hg,envs]" --pre
         shell: pwsh
 
       - name: Install asv

.github/workflows/triggered.yml

@@ -44,7 +44,7 @@ jobs:
         uses: browser-actions/setup-chrome@latest
 
       - name: Install dependencies
-        run: python -m pip install ".[test,hg]"
+        run: python -m pip install ".[test,hg,envs]"
 
       - name: Get asv_runner to be tested
         uses: actions/checkout@v4

asv/commands/profiling.py

@@ -4,15 +4,14 @@
 import io
 import os
 import pstats
-import sys
 import tempfile
 
 from asv_runner.console import color_print
 
 from . import Command, common_args
 from ..benchmarks import Benchmarks
 from ..console import log
-from ..environment import get_environments, is_existing_only
+from ..environment import get_environments, is_existing_only, ExistingEnvironment
 from ..machine import Machine
 from ..profiling import ProfilerGui
 from ..repo import get_repo, NoSuchNameError
@@ -168,13 +167,13 @@ def run(cls, conf, benchmark, revision=None, gui=None, output=None,
                             "using an existing environment.")
 
             if env is None:
-                # Fallback
-                env = environments[0]
-
-            if env.python != "{0}.{1}".format(*sys.version_info[:2]):
-                raise util.UserError(
-                    "Profiles must be run in the same version of Python as the "
-                    "asv main process")
+                # Fallback, first valid python environment
+                env = [
+                    env
+                    for env in environments
+                    if util.env_py_is_sys_version(env.python)
+                    or isinstance(env, ExistingEnvironment)
+                ][0]
 
             benchmarks = Benchmarks.discover(conf, repo, environments,
                                              [commit_hash],

asv/environment.py

@@ -423,16 +423,17 @@ def get_environment_class(conf, python):
     if python == 'same':
         return ExistingEnvironment
 
-    # Try the subclasses in reverse order so custom plugins come first
-    classes = list(util.iter_subclasses(Environment))[::-1]
+    classes = list(util.iter_subclasses(Environment))
 
     if conf.environment_type:
         cls = get_environment_class_by_name(conf.environment_type)
         classes.remove(cls)
         classes.insert(0, cls)
+    else:
+        raise RuntimeError("Environment type must be specified")
 
     for cls in classes:
-        if cls.matches_python_fallback and cls.matches(python):
+        if cls.matches_python_fallback or cls.matches(python):
             return cls
     raise EnvironmentUnavailable(
         f"No way to create environment for python='{python}'")
@@ -501,6 +502,7 @@ def __init__(self, conf, python, requirements, tagged_env_vars):
 
         """
         self._env_dir = conf.env_dir
+        self._python = python
         self._repo_subdir = conf.repo_subdir
         self._install_timeout = conf.install_timeout  # gh-391
         self._default_benchmark_timeout = conf.default_benchmark_timeout # gh-973

asv/plugin_manager.py

@@ -1,13 +1,21 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
 
 import sys
+import re
 import pkgutil
 import importlib
 
 from . import commands, plugins
 from .console import log
 
-ENV_PLUGINS = [".mamba", ".virtualenv", ".conda", ".rattler"]
+ENV_PLUGIN_REGEXES = [
+    r"\.mamba$",
+    r"\._mamba_helpers$",
+    r"\.virtualenv$",
+    r"\.conda$",
+    r"\.rattler$",
+]
+
 
 class PluginManager:
     """
@@ -25,20 +33,24 @@ def __init__(self):
 
     def load_plugins(self, package):
         prefix = package.__name__ + "."
-        for module_finder, name, ispkg in pkgutil.iter_modules(package.__path__, prefix):
+        for module_finder, name, ispkg in pkgutil.iter_modules(
+            package.__path__, prefix
+        ):
             try:
                 mod = importlib.import_module(name)
                 self.init_plugin(mod)
                 self._plugins.append(mod)
             except ModuleNotFoundError as err:
-                if any(keyword in name for keyword in ENV_PLUGINS):
+                if any(re.search(regex, name) for regex in ENV_PLUGIN_REGEXES):
                     continue  # Fine to not have these
                 else:
                     log.error(f"Couldn't load {name} because\n{err}")
 
     def _load_plugin_by_name(self, name):
         prefix = plugins.__name__ + "."
-        for module_finder, module_name, ispkg in pkgutil.iter_modules(plugins.__path__, prefix):
+        for module_finder, module_name, ispkg in pkgutil.iter_modules(
+            plugins.__path__, prefix
+        ):
             if name in module_name:
                 mod = importlib.import_module(module_name)
                 return mod

asv/plugins/conda.py

@@ -133,7 +133,7 @@ def _setup(self):
 
         # Changed in v0.6.5, gh-1294
         # previously, the user provided environment was assumed to handle the python version
-        conda_args = [util.replace_python_version(arg, self._python) for arg in conda_args]
+        conda_args = [util.replace_cpython_version(arg, self._python) for arg in conda_args]
 
         if not self._conda_environment_file:
             conda_args = ['wheel', 'pip'] + conda_args

asv/plugins/mamba.py

@@ -158,7 +158,7 @@ def _setup(self):
         # Changed in v0.6.5, gh-1294
         # previously, the user provided environment was assumed to handle the python version
         mamba_pkgs = [
-            util.replace_python_version(pkg, self._python) for pkg in mamba_pkgs
+            util.replace_cpython_version(pkg, self._python) for pkg in mamba_pkgs
         ]
         self.context.prefix_params.target_prefix = self._path
         solver = MambaSolver(

asv/plugins/rattler.py

@@ -96,7 +96,7 @@ async def _async_setup(self):
                 except KeyError:
                     raise KeyError("Only pip is supported as a secondary key")
         _pkgs += _args
-        _pkgs = [util.replace_python_version(pkg, self._python) for pkg in _pkgs]
+        _pkgs = [util.replace_cpython_version(pkg, self._python) for pkg in _pkgs]
         solved_records = await solve(
             # Channels to use for solving
             channels=self._channels,

asv/template/asv.conf.json

@@ -49,7 +49,7 @@
     // "dvcs": "git",
 
     // The tool to use to create environments.  May be "conda",
-    // "virtualenv", "mamba" (above 3.8)
+    // "virtualenv", "mamba" or "rattler" (above 3.8)
     // or other value depending on the plugins in use.
     // If missing or the empty string, the tool will be automatically
     // determined by looking for tools on the PATH environment

asv/util.py

@@ -1427,14 +1427,30 @@ def get_matching_environment(environments, result=None):
             env
             for env in environments
             if (result is None or result.env_name == env.name)
-            and env.python == "{0}.{1}".format(*sys.version_info[:2])
+            and env_py_is_sys_version(env.python)
         ),
         None,
     )
 
-def replace_python_version(arg, new_version):
-    match = re.match(r'^python(\W|$)', arg)
+
+def replace_cpython_version(arg, new_version):
+    match = re.match(r"^python(\W|$)", arg)
     if match and not match.group(1).isalnum():
         return f"python={new_version}"
     else:
         return arg
+
+
+def extract_cpython_version(env_python):
+    version_regex = r"(\d+\.\d+)$"
+    match = re.search(version_regex, env_python)
+    if match:
+        return match.group(1)
+    else:
+        return None
+
+
+def env_py_is_sys_version(env_python):
+    return extract_cpython_version(env_python) == "{0}.{1}".format(
+        *sys.version_info[:2]
+    )

changelog.d/1446.bugfix.rst

@@ -0,0 +1 @@
+Environment types can be specified for pytest

test/conftest.py

@@ -46,8 +46,8 @@ def pytest_addoption(parser):
     parser.addoption(
         "--runflaky", action="store_true", default=False, help="run flaky tests"
     )
-    parser.addoption("--environment-type", action="store", default=None,
-                     choices=("conda", "virtualenv", "mamba"),
+    parser.addoption("--environment-type", action="store", default="virtualenv",
+                     choices=("conda", "virtualenv", "mamba", "rattler"),
                      help="environment_type to use in tests by default")
 
 
@@ -75,26 +75,29 @@ def generate_basic_conf(tmpdir,
     # values not in test_dev.py copy
     repo_path = tools.generate_test_repo(tmpdir, values,
                                          subdir=repo_subdir).path
+    global env_type
 
     conf_dict = {
         'env_dir': 'env',
         'benchmark_dir': 'benchmark',
         'results_dir': 'results_workflow',
         'html_dir': 'html',
         'repo': relpath(repo_path),
+        'environment_type': env_type,
         'project': 'asv',
+        'conda_channels': ["conda-forge"],
         'dvcs': 'git',
         'matrix': {
-            "asv-dummy-test-package-1": [None],
-            "asv-dummy-test-package-2": tools.DUMMY2_VERSIONS,
+            "pip+asv-dummy-test-package-1": [None],
+            "pip+asv-dummy-test-package-2": tools.DUMMY2_VERSIONS,
         },
     }
     if not dummy_packages:
         conf_dict['matrix'] = {}
     elif conf_version == 2:
         conf_dict['matrix'] = {
-            "asv_dummy_test_package_1": [""],
-            "asv_dummy_test_package_2": tools.DUMMY2_VERSIONS,
+            "pip+asv_dummy_test_package_1": [""],
+            "pip+asv_dummy_test_package_2": tools.DUMMY2_VERSIONS,
         }
     if repo_subdir:
         conf_dict['repo_subdir'] = repo_subdir
@@ -111,6 +114,8 @@ def pytest_sessionstart(session):
     _monkeypatch_conda_lock(session.config)
 
     # Unregister unwanted environment types
+    # XXX: Ugly hack to get the variable into generate_basic_conf
+    global env_type
     env_type = session.config.getoption('environment_type')
     if env_type is not None:
         import asv.environment
@@ -343,7 +348,7 @@ def basic_html(request):
 
 
 @pytest.fixture
-def benchmarks_fixture(tmpdir):
+def benchmarks_fixture(tmpdir, request: pytest.FixtureRequest):
     tmpdir = str(tmpdir)
     os.chdir(tmpdir)
 
@@ -353,6 +358,8 @@ def benchmarks_fixture(tmpdir):
     d.update(ASV_CONF_JSON)
     d['env_dir'] = "env"
     d['benchmark_dir'] = 'benchmark'
+    d['environment_type'] = request.config.getoption('environment_type')
+    d['conda_channels'] = ["conda-forge"]
     d['repo'] = tools.generate_test_repo(tmpdir, [0]).path
     d['branches'] = ["master"]
     conf = config.Config.from_json(d)
@@ -435,3 +442,15 @@ def pytest_collection_modifyitems(config, items):
     for item in items:
         if "flaky" in item.keywords:
             item.add_marker(skip_flaky)
+
+
+@pytest.fixture
+def skip_virtualenv(request: pytest.FixtureRequest):
+    if request.config.getoption('environment_type') == 'virtualenv':
+        pytest.skip('Cannot run this test with virtualenv')
+
+
+@pytest.fixture
+def skip_no_conda(request: pytest.FixtureRequest):
+    if request.config.getoption('environment_type') != 'conda':
+        pytest.skip('Needs to be run with conda')

test/example_plugin.py

@@ -4,4 +4,22 @@
 
 
 class MyEnvironment(Environment):
-    pass
+    tool_name = "myenv"
+    def __init__(self, conf, python, requirements, tagged_env_vars):
+        """
+        Parameters
+        ----------
+        conf : Config instance
+
+        python : str
+            Version of Python.  Must be of the form "MAJOR.MINOR".
+
+        requirements : dict
+            Dictionary mapping a PyPI package name to a version
+            identifier string.
+        """
+        self._python = python
+        self._requirements = requirements
+        self._channels = conf.conda_channels
+        self._environment_file = None
+        super(MyEnvironment, self).__init__(conf, python, requirements, tagged_env_vars)