From b2eccd208fdab73cdd0d546169a58f0771bda95f Mon Sep 17 00:00:00 2001 From: Stephen Fleming Date: Mon, 24 Apr 2023 12:47:30 -0400 Subject: [PATCH] Initial attempt to bump to python 3.8 --- .github/workflows/run_pytest.yml | 2 +- README.rst | 6 ++-- .../tests/benchmarking/cuda_check_inputs.json | 2 +- .../docker_image_check_cuda_status.wdl | 2 ++ .../tests/test_checkpoint.py | 29 ++++++++++++------- docker/Dockerfile | 4 +-- docker/DockerfileGit | 2 +- docs/source/installation/index.rst | 4 +-- setup.py | 2 +- 9 files changed, 31 insertions(+), 22 deletions(-) diff --git a/.github/workflows/run_pytest.yml b/.github/workflows/run_pytest.yml index 37e652b..07995a1 100644 --- a/.github/workflows/run_pytest.yml +++ b/.github/workflows/run_pytest.yml @@ -10,7 +10,7 @@ jobs: runs-on: 'ubuntu-latest' strategy: matrix: - python-version: ['3.7'] + python-version: ['3.8'] steps: - name: 'Checkout repo' diff --git a/README.rst b/README.rst index ea39e2c..effccde 100644 --- a/README.rst +++ b/README.rst @@ -45,8 +45,8 @@ CellBender can be installed via $ pip install cellbender -(and we recommend installing in its own ``conda`` environment to prevent -conflicts with other software). +(and we recommend installing in its own ``conda`` environment, using python 3.8, +to prevent conflicts with other software). CellBender is run as a command-line tool, as in @@ -98,7 +98,7 @@ Create a conda environment and activate it: .. code-block:: console - $ conda create -n cellbender python=3.7 + $ conda create -n cellbender python=3.8 $ conda activate cellbender Install the `pytables `_ module: diff --git a/cellbender/remove_background/tests/benchmarking/cuda_check_inputs.json b/cellbender/remove_background/tests/benchmarking/cuda_check_inputs.json index a92c41a..d852484 100644 --- a/cellbender/remove_background/tests/benchmarking/cuda_check_inputs.json +++ b/cellbender/remove_background/tests/benchmarking/cuda_check_inputs.json @@ -1,4 +1,4 @@ { - "check_pytorch_cuda_status.run_check_pytorch_cuda_status.docker_image": "us.gcr.io/broad-dsde-methods/cellbender:20230414" + "check_pytorch_cuda_status.run_check_pytorch_cuda_status.docker_image": "us.gcr.io/broad-dsde-methods/cellbender:20230424" } diff --git a/cellbender/remove_background/tests/benchmarking/docker_image_check_cuda_status.wdl b/cellbender/remove_background/tests/benchmarking/docker_image_check_cuda_status.wdl index 8763bfe..bf440c6 100644 --- a/cellbender/remove_background/tests/benchmarking/docker_image_check_cuda_status.wdl +++ b/cellbender/remove_background/tests/benchmarking/docker_image_check_cuda_status.wdl @@ -20,6 +20,8 @@ task run_check_pytorch_cuda_status { import torch assert torch.cuda.is_available() CODE + + cellbender remove-background -h } runtime { docker: "${docker_image}" diff --git a/cellbender/remove_background/tests/test_checkpoint.py b/cellbender/remove_background/tests/test_checkpoint.py index 45cb693..80d30d7 100644 --- a/cellbender/remove_background/tests/test_checkpoint.py +++ b/cellbender/remove_background/tests/test_checkpoint.py @@ -6,7 +6,7 @@ import torch from torch.distributions import constraints import pyro -import pyro.optim as optim +import pyro.optim import cellbender from cellbender.remove_background.checkpoint import make_tarball, unpack_tarball, \ @@ -261,7 +261,7 @@ def test_save_and_load_pyro_checkpoint(tmpdir_factory, batch_size_n): initial_model = PyroModel(dim=dim) # set up the inference process - scheduler = optim.ClippedAdam({'lr': lr, 'clip_norm': 10.}) + scheduler = pyro.optim.ClippedAdam({'lr': lr, 'clip_norm': 10.}) svi = pyro.infer.SVI(initial_model.model, initial_model.guide, scheduler, loss=pyro.infer.Trace_ELBO()) w1 = _get_params(initial_model.encoder) @@ -316,7 +316,7 @@ def test_save_and_load_pyro_checkpoint(tmpdir_factory, batch_size_n): # one-shot training straight through model_one_shot = PyroModel(dim=dim) # resets random state - scheduler = optim.ClippedAdam({'lr': lr, 'clip_norm': 10.}) + scheduler = pyro.optim.ClippedAdam({'lr': lr, 'clip_norm': 10.}) train_loader = new_train_loader(data=dataset, batch_size=batch_size_n) svi_one_shot = pyro.infer.SVI(model_one_shot.model, model_one_shot.guide, scheduler, loss=pyro.infer.Trace_ELBO()) model_one_shot.loss.extend(train_pyro(n_epochs=epochs, data_loader=train_loader, svi=svi_one_shot)) @@ -526,13 +526,20 @@ def test_save_and_load_cellbender_checkpoint(tmpdir_factory, cuda, scheduler): # 'Guide traces disagree with and without checkpoint restart' +# def ExponentialLR(optim_args) -> pyro.optim.PyroLRScheduler: +# """ +# Wraps :class:`pyro.optim.clipped_adam.ClippedAdam` with :class:`~pyro.optim.optim.PyroOptim`. +# """ +# return pyro.optim.PyroLRScheduler(torch.optim.lr_scheduler.ExponentialLR, optim_args) + + @pytest.mark.parametrize( "Optim, config", [ - (optim.ClippedAdam, {"lr": 0.01}), - (optim.ExponentialLR, {"optimizer": torch.optim.SGD, - "optim_args": {"lr": 0.01}, - "gamma": 0.9}), + (pyro.optim.ClippedAdam, {"lr": 0.01}), + (pyro.optim.ExponentialLR, {"optimizer": torch.optim.SGD, + "optim_args": {"lr": 0.01}, + "gamma": 0.9}), ], ) def test_optimizer_checkpoint_restart(Optim, config, tmpdir_factory): @@ -557,7 +564,7 @@ def guide(): def get_snapshot(optimizer): s = {k: v.data.clone() for k, v in store.items()} - if type(optimizer) == optim.lr_scheduler.PyroLRScheduler: + if type(optimizer) == pyro.optim.lr_scheduler.PyroLRScheduler: lr = list(optimizer.optim_objs.values())[0].get_last_lr()[0] else: lr = list(optimizer.optim_objs.values())[0].param_groups[0]['lr'] @@ -573,7 +580,7 @@ def get_snapshot(optimizer): for _ in range(5 + 10): svi.step() expected.append(get_snapshot(optimizer)) - if type(optimizer) == optim.lr_scheduler.PyroLRScheduler: + if type(optimizer) == pyro.optim.lr_scheduler.PyroLRScheduler: svi.optim.step() del svi, optimizer @@ -586,7 +593,7 @@ def get_snapshot(optimizer): for _ in range(5): svi.step() actual.append(get_snapshot(optimizer)) - if type(optimizer) == optim.lr_scheduler.PyroLRScheduler: + if type(optimizer) == pyro.optim.lr_scheduler.PyroLRScheduler: svi.optim.step() # checkpoint @@ -605,7 +612,7 @@ def get_snapshot(optimizer): for _ in range(10): svi.step() actual.append(get_snapshot(optimizer)) - if type(optimizer) == optim.lr_scheduler.PyroLRScheduler: + if type(optimizer) == pyro.optim.lr_scheduler.PyroLRScheduler: svi.optim.step() # display learning rates and actual/expected values for z_loc diff --git a/docker/Dockerfile b/docker/Dockerfile index 91c081a..a8b5e08 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi && apt-get clean \ && sudo rm -rf /var/lib/apt/lists/* \ # get miniconda - && curl -so $HOME/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py37_23.1.0-1-Linux-x86_64.sh \ + && curl -so $HOME/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_23.1.0-1-Linux-x86_64.sh \ && chmod +x $HOME/miniconda.sh \ && $HOME/miniconda.sh -b -p $CONDA_DIR \ && rm $HOME/miniconda.sh \ @@ -30,6 +30,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi # get compiled crcmod for gsutil && conda install -y -c conda-forge crcmod \ # install cellbender and its dependencies - && yes | pip install /software/cellbender/ \ + && yes | pip install -e /software/cellbender/ \ && conda clean -yaf \ && sudo rm -rf ~/.cache/pip diff --git a/docker/DockerfileGit b/docker/DockerfileGit index 05bbde4..810a46a 100644 --- a/docker/DockerfileGit +++ b/docker/DockerfileGit @@ -17,7 +17,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi && apt-get clean \ && sudo rm -rf /var/lib/apt/lists/* \ # get miniconda - && curl -so $HOME/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py37_23.1.0-1-Linux-x86_64.sh \ + && curl -so $HOME/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_23.1.0-1-Linux-x86_64.sh \ && chmod +x $HOME/miniconda.sh \ && $HOME/miniconda.sh -b -p $CONDA_DIR \ && rm $HOME/miniconda.sh \ diff --git a/docs/source/installation/index.rst b/docs/source/installation/index.rst index 4defef3..e3522d7 100644 --- a/docs/source/installation/index.rst +++ b/docs/source/installation/index.rst @@ -26,7 +26,7 @@ packages you may have installed. .. code-block:: console - $ conda create -n cellbender python=3.7 + $ conda create -n cellbender python=3.8 $ conda activate cellbender (cellbender) $ pip install cellbender @@ -38,7 +38,7 @@ Create a conda environment and activate it: .. code-block:: console - $ conda create -n cellbender python=3.7 + $ conda create -n cellbender python=3.8 $ conda activate cellbender Install the `pytables `_ module: diff --git a/setup.py b/setup.py index 23fa76d..9a5d6ff 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ def get_version() -> str: 'Development Status :: 4 - Beta', 'Intended Audience :: Science/Research', 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', 'Topic :: Scientific/Engineering :: Bio-Informatics', ], keywords='scRNA-seq bioinformatics',