From f2f605e687e7bda210a88d0353e27b9c1ed8ed65 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 10:25:23 -0700 Subject: [PATCH 01/11] add github workflow --- .github/workflows/python-package.yml | 126 +++++++++++++++++++++++++++ requirements.txt | 2 +- 2 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/python-package.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 00000000..c6f8d4ab --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,126 @@ +# Workflow is based on the Astropy GitHub actions workflow, ci_workflows.yml +name: CI + +on: + push: + branches: + - '*' + tags: + - '*' + pull_request: + +jobs: + tests: + name: Standard Python tests. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + python-version: [3.6, 3.7, 3.8] + astropy-version: ['<4.0', '<4.1', '>4.0'] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi + pip install -U "astropy${{ matrix.astropy-version }}" + - name: Run the test + run: | + pytest + + coverage: + name: Code coverage test. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + os: [ubuntu-latest] + python-version: [3.8] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pytest-cov coveralls + if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi + - name: Run the test with coverage + run: | + pytest --cov + - name: Coveralls + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + coveralls + + docs: + name: Check Sphinx documentation. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: [3.8] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install Sphinx + - name: Test the documentation + run: | + sphinx-build -W --keep-going -b html doc doc/_build/html + + style: + name: Check code style; allow failures. + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: [3.8] + + steps: + - name: Checkout code + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pycodestyle + - name: Test the style + # This is equivalent to an allowed falure. + continue-on-error: true + run: | + pycodestyle --count py/desiutil diff --git a/requirements.txt b/requirements.txt index 99d973d4..2ca9b70d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ setuptools requests pyyaml -astropy==2.0.16 +astropy healpy==1.12.9 matplotlib From b933bdab0446b7a389c987bd541b44f2c9a2fb9e Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 10:57:19 -0700 Subject: [PATCH 02/11] remove some references to "master" --- .github/workflows/python-package.yml | 2 +- bin/desiBootstrap.sh | 9 +++++---- doc/desiInstall.rst | 12 ++++++++---- doc/index.rst | 2 +- requirements.txt | 2 +- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index c6f8d4ab..267ecb6a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: matrix: os: [ubuntu-latest] python-version: [3.6, 3.7, 3.8] - astropy-version: ['<4.0', '<4.1', '>4.0'] + astropy-version: ['<3.0', '<4.1'] #, '>4.0'] steps: - name: Checkout code diff --git a/bin/desiBootstrap.sh b/bin/desiBootstrap.sh index 962a1189..acc74c91 100755 --- a/bin/desiBootstrap.sh +++ b/bin/desiBootstrap.sh @@ -85,15 +85,16 @@ fi # # Export # -[[ -n "${verbose}" ]] && echo git clone https://github.com/desihub/desiutil.git desiutil-master -git clone https://github.com/desihub/desiutil.git desiutil-master +checkout=desiutil-checkout +[[ -n "${verbose}" ]] && echo git clone https://github.com/desihub/desiutil.git ${checkout} +git clone https://github.com/desihub/desiutil.git ${checkout} if [[ -n "${branch}" ]]; then - cd desiutil-master + cd ${checkout} [[ -n "${verbose}" ]] && echo git checkout ${branch} git checkout ${branch} cd .. fi -export DESIUTIL=$(pwd)/desiutil-master +export DESIUTIL=$(pwd)/${checkout} export PATH=${DESIUTIL}/bin:${PATH} if [[ -z "${PYTHONPATH}" ]]; then export PYTHONPATH=${DESIUTIL}/py diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index bf3756b3..2f4cea37 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -149,7 +149,7 @@ Because of the structures of the DESI code repositories, it is sometimes necessa to specify a directory name along with the product name. desiInstall contains a list of known products, but it is not necessarily complete. desiInstall parses the input to determine the base name and base version to install. At this -stage desiInstall also determines whether a trunk or branch install has +stage desiInstall also determines whether a branch install [#]_ has been requested. The internal list of known products can be added to or overridden on the @@ -162,6 +162,10 @@ command line:: The ``-p`` option can be specified multiple times, though in practice, it only matters to the product actually being installed. +.. [#] In this document, "branch" refers to anything that is not a tagged version. + This could include default branches such as "trunk" in Subversion repositories, + or any default branch in a git repository. + Product Existence ----------------- @@ -174,8 +178,8 @@ Download Code ------------- The code is downloaded, using :command:`svn export` for standard (tag) installs, or -:command:`svn checkout` for trunk or branch installs. For GitHub installs, desiInstall -will look for a release tarball, or do a :command:`git clone` for tag or master/branch +:command:`svn checkout` for branch installs. For GitHub installs, desiInstall +will look for a release tarball, or do a :command:`git clone` for tag or branch installs. desiInstall will set the environment variable :envvar:`WORKING_DIR` to point to the directory containing this downloaded code. @@ -272,7 +276,7 @@ added to Python's :data:`sys.path`. Can We Just Copy the Download? ------------------------------ -If the build-type is *only* 'plain', or if a trunk or branch install is +If the build-type is *only* 'plain', or if a branch install is requested, the downloaded code will be copied to :envvar:`INSTALL_DIR`. Further Python or C/C++ install steps described below will be skipped. diff --git a/doc/index.rst b/doc/index.rst index e0b05b7f..496dec3a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -18,7 +18,7 @@ Required Dependencies These packages must be installed for desiutil to work properly: * `pyyaml `_ -* `requests `_ +* `requests `_ * `astropy `_ - This implies a dependency on `NumPy `_ diff --git a/requirements.txt b/requirements.txt index 2ca9b70d..0592c695 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ setuptools requests pyyaml -astropy +astropy<4.1 healpy==1.12.9 matplotlib From 442bd0d9ee2c331884ac4b130322b2c081edff78 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 11:06:10 -0700 Subject: [PATCH 03/11] fix healpy version --- .github/workflows/python-package.yml | 8 ++++---- requirements.txt | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 267ecb6a..fe83ef82 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -31,7 +31,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi pip install -U "astropy${{ matrix.astropy-version }}" @@ -59,7 +59,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel python -m pip install pytest pytest-cov coveralls if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi - name: Run the test with coverage @@ -91,7 +91,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel python -m pip install Sphinx - name: Test the documentation run: | @@ -117,7 +117,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Python dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip wheel python -m pip install pycodestyle - name: Test the style # This is equivalent to an allowed falure. diff --git a/requirements.txt b/requirements.txt index 0592c695..11f5272f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,5 @@ setuptools requests pyyaml astropy<4.1 -healpy==1.12.9 +healpy matplotlib From ce1a4f97aa017fb97ce20a604cae72273622cc6f Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 11:14:57 -0700 Subject: [PATCH 04/11] fix healpy version --- .github/workflows/python-package.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index fe83ef82..081b5e6f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -35,6 +35,7 @@ jobs: python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi pip install -U "astropy${{ matrix.astropy-version }}" + if [ "${{ matrix.astropy-version }}" = "<3.0" ]; pip install -U "healpy<1.13"; fi - name: Run the test run: | pytest From ac418bd7d566756e33c854b7e81c5006e6ae1819 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 11:18:39 -0700 Subject: [PATCH 05/11] syntax error --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 081b5e6f..5d01f726 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -35,7 +35,7 @@ jobs: python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi pip install -U "astropy${{ matrix.astropy-version }}" - if [ "${{ matrix.astropy-version }}" = "<3.0" ]; pip install -U "healpy<1.13"; fi + if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy<1.13"; fi - name: Run the test run: | pytest From 0e9deba1c26c3dc6b95b2630ca19745c58f66949 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 11:29:21 -0700 Subject: [PATCH 06/11] update docstring --- py/desiutil/install.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 249fc44d..9e7ec6d7 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -305,13 +305,12 @@ def get_product_version(self): return (self.fullproduct, self.baseproduct, self.baseversion) def identify_branch(self): - """If this is not a tag install, determine whether this is a trunk - or branch install. + """If this is not a tag install, determine the branch identity. Returns ------- :class:`str` - The full path to the branch/tag/trunk/master code. + The full path to the branch code. """ self.is_branch = self.options.product_version.startswith('branches') self.is_trunk = (self.options.product_version == 'trunk' or From 09dd30f180710d31d9bd83743459411fdd38830f Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 11:33:25 -0700 Subject: [PATCH 07/11] drop astropy 2 support --- .github/workflows/python-package.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 5d01f726..e5b92747 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: matrix: os: [ubuntu-latest] python-version: [3.6, 3.7, 3.8] - astropy-version: ['<3.0', '<4.1'] #, '>4.0'] + # astropy-version: ['<3.0', '<4.1'] #, '>4.0'] steps: - name: Checkout code @@ -34,8 +34,8 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi - pip install -U "astropy${{ matrix.astropy-version }}" - if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy<1.13"; fi +# pip install -U "astropy${{ matrix.astropy-version }}" +# if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy<1.13"; fi - name: Run the test run: | pytest From 34482fd8b30878dcf90c3b1ec22eb4b6c49c9afe Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 15:18:43 -0700 Subject: [PATCH 08/11] explicitly denote all branches, including default branches --- .github/workflows/python-package.yml | 6 +- README.rst | 35 +++++----- doc/desiInstall.rst | 22 ++++++- py/desiutil/install.py | 97 ++++++++++++++-------------- py/desiutil/test/test_install.py | 2 +- 5 files changed, 87 insertions(+), 75 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index e5b92747..8ab52e66 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: matrix: os: [ubuntu-latest] python-version: [3.6, 3.7, 3.8] - # astropy-version: ['<3.0', '<4.1'] #, '>4.0'] + astropy-version: ['<3.0', '<4.1'] #, '>4.0'] steps: - name: Checkout code @@ -34,8 +34,8 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi -# pip install -U "astropy${{ matrix.astropy-version }}" -# if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy<1.13"; fi + pip install -U "astropy${{ matrix.astropy-version }}" + if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy==1.12.9"; fi - name: Run the test run: | pytest diff --git a/README.rst b/README.rst index 72040266..3002a853 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,20 @@ desiutil ======== +|Actions Status| |Coveralls Status| |Documentation Status| + +.. |Actions Status| image:: https://github.com/desihub/desiutil/workflows/CI/badge.svg + :target: https://github.com/desihub/desiutil/actions + :alt: GitHub Actions CI Status + +.. |Coveralls Status| image:: https://coveralls.io/repos/desihub/desiutil/badge.svg + :target: https://coveralls.io/github/desihub/desiutil + :alt: Test Coverage Status + +.. |Documentation Status| image:: https://readthedocs.org/projects/desiutil/badge/?version=latest + :target: https://desiutil.readthedocs.io/en/latest/ + :alt: Documentation Status + Introduction ============ @@ -13,26 +27,7 @@ Full Documentation Please visit `desiutil on Read the Docs`_ -.. image:: https://readthedocs.org/projects/desiutil/badge/?version=latest - :target: http://desiutil.readthedocs.org/en/latest/ - :alt: Documentation Status - -.. _`desiutil on Read the Docs`: http://desiutil.readthedocs.org/en/latest/ - -Travis Build Status -=================== - -.. image:: https://img.shields.io/travis/desihub/desiutil.svg - :target: https://travis-ci.org/desihub/desiutil - :alt: Travis Build Status - - -Test Coverage Status -==================== - -.. image:: https://coveralls.io/repos/desihub/desiutil/badge.svg?service=github - :target: https://coveralls.io/github/desihub/desiutil - :alt: Test Coverage Status +.. _`desiutil on Read the Docs`: https://desiutil.readthedocs.io/en/latest/ License ======= diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index 2f4cea37..a732b272 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -7,14 +7,32 @@ Introduction This document describes the desiInstall process and the logic behind it. -The primary purpose of desiInstall is to install DESI software **at NERSC**. +The primary purpose of :command:`desiInstall` is to install DESI software **at NERSC**. Using it to install software at locations other than NERSC is theoretically possible, but not supported. +Basic Invocation of desiInstall +=============================== + +:command:`desiInstall` is invoked with the product and version to be installed:: + + desiInstall desiutil 3.0.3 + +The version should correspond to a tag in the repository corresponding to +the product. + +Branches can also be installed, and the resulting install will be a +checkout of the repository, set to the requested branch. This is specified +by prepending ``branches/`` to the version name:: + + desiInstall desiutil branches/branch-compile + +Finally, for Subversion repositories, ``trunk`` is a shorthand for ``branches/trunk``. + Configuring desiInstall ======================= -desiInstall has many options, which are best viewed by typing +:command:`desiInstall` has many options, which are best viewed by typing ``desiInstall --help``. In addition, desiInstall both reads and sets several environment variables. diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 9e7ec6d7..047bfa99 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -116,8 +116,6 @@ class DesiInstall(object): ``True`` if the selected product lives on GitHub. is_branch : :class:`bool` ``True`` if a branch has been selected. - is_trunk : :class:`bool` - ``True`` if trunk or the master branch has been selected. log : :class:`logging.Logger` Logging object. nersc : :class:`str` @@ -312,15 +310,17 @@ def identify_branch(self): :class:`str` The full path to the branch code. """ - self.is_branch = self.options.product_version.startswith('branches') - self.is_trunk = (self.options.product_version == 'trunk' or - self.options.product_version == 'master') - if self.is_trunk or self.is_branch: + self.is_branch = (self.options.product_version.startswith('branches') or + self.options.product_version == 'trunk') + if self.is_branch: if self.github: self.product_url = self.fullproduct + '.git' else: - self.product_url = os.path.join(self.fullproduct, - self.options.product_version) + if self.options.product_version == 'branches/trunk': + self.product_url = os.path.join(self.fullproduct, 'trunk') + else: + self.product_url = os.path.join(self.fullproduct, + self.options.product_version) else: if self.github: self.product_url = os.path.join(self.fullproduct, 'archive', @@ -396,20 +396,19 @@ def get_code(self): if not self.options.test: shutil.rmtree(self.working_dir) if self.github: - if self.is_trunk or self.is_branch: - if self.is_branch: - try: - r = requests.get(os.path.join(self.fullproduct, 'tree', - self.baseversion)) - r.raise_for_status() - except requests.exceptions.HTTPError: - message = ("Branch {0} does not appear to exist. " + - "HTTP response was {1:d}.").format( - self.baseversion, r.status_code) - self.log.critical(message) - raise DesiInstallException(message) - command = ['git', 'clone', '-q', self.product_url, - self.working_dir] + if self.is_branch: + try: + r = requests.get(os.path.join(self.fullproduct, 'tree', + self.baseversion)) + r.raise_for_status() + except requests.exceptions.HTTPError: + message = ("Branch {0} does not appear to exist. " + + "HTTP response was {1:d}.").format( + self.baseversion, r.status_code) + self.log.critical(message) + raise DesiInstallException(message) + command = ['git', 'clone', '-q', '-b', self.baseversion, + self.product_url, self.working_dir] self.log.debug(' '.join(command)) if self.options.test: out, err = 'Test Mode.', '' @@ -423,29 +422,29 @@ def get_code(self): err) self.log.critical(message) raise DesiInstallException(message) - if self.is_branch: - original_dir = os.getcwd() - self.log.debug("os.chdir('%s')", self.working_dir) - if not self.options.test: - os.chdir(self.working_dir) - command = ['git', 'checkout', '-q', '-b', self.baseversion, - 'origin/'+self.baseversion] - self.log.debug(' '.join(command)) - if self.options.test: - out, err = 'Test Mode.', '' - else: - proc = Popen(command, universal_newlines=True, - stdout=PIPE, stderr=PIPE) - out, err = proc.communicate() - self.log.debug(out) - if len(err) > 0: - message = ("git error while changing branch:" + - " {0}".format(err)) - self.log.critical(message) - raise DesiInstallException(message) - self.log.debug("os.chdir('%s')", original_dir) - if not self.options.test: - os.chdir(original_dir) + # if self.is_branch: + # original_dir = os.getcwd() + # self.log.debug("os.chdir('%s')", self.working_dir) + # if not self.options.test: + # os.chdir(self.working_dir) + # command = ['git', 'checkout', '-q', '-b', self.baseversion, + # 'origin/'+self.baseversion] + # self.log.debug(' '.join(command)) + # if self.options.test: + # out, err = 'Test Mode.', '' + # else: + # proc = Popen(command, universal_newlines=True, + # stdout=PIPE, stderr=PIPE) + # out, err = proc.communicate() + # self.log.debug(out) + # if len(err) > 0: + # message = ("git error while changing branch:" + + # " {0}".format(err)) + # self.log.critical(message) + # raise DesiInstallException(message) + # self.log.debug("os.chdir('%s')", original_dir) + # if not self.options.test: + # os.chdir(original_dir) else: if self.options.test: self.log.debug("Test Mode. Skipping download of %s.", @@ -480,7 +479,7 @@ def get_code(self): self.log.critical(message) raise DesiInstallException(message) else: - if self.is_trunk or self.is_branch: + if self.is_branch: get_svn = 'checkout' else: get_svn = 'export' @@ -672,7 +671,7 @@ def install_module(self): """ dev = False if 'py' in self.build_type: - if self.is_trunk or self.is_branch: + if self.is_branch: dev = True else: if os.path.isdir(os.path.join(self.working_dir, 'py')): @@ -749,7 +748,7 @@ def prepare_environment(self): def install(self): """Run setup.py, etc. """ - if (self.build_type == set(['plain']) or self.is_trunk or self.is_branch): + if (self.build_type == set(['plain']) or self.is_branch): # # For certain installs, all that is needed is to copy the # downloaded code to the install directory. @@ -938,7 +937,7 @@ def permissions(self): self.log.debug(out) # Remove write permission to avoid accidental changes - if self.is_trunk or self.is_branch: + if self.is_branch: chmod_mode = 'g-w,o-w' else: chmod_mode = 'a-w' diff --git a/py/desiutil/test/test_install.py b/py/desiutil/test/test_install.py index b0007494..ae0642b8 100644 --- a/py/desiutil/test/test_install.py +++ b/py/desiutil/test/test_install.py @@ -169,7 +169,7 @@ def test_identify_branch(self): self.assertEqual(url, ('https://github.com/desihub/desiutil/archive/' + '1.0.0.tar.gz')) - options = self.desiInstall.get_options(['desiutil', 'master']) + options = self.desiInstall.get_options(['desiutil', 'branches/master']) out = self.desiInstall.get_product_version() url = self.desiInstall.identify_branch() self.assertEqual(url, From fdb7d5776bd80d38892c9c80a11e8c2daef7535d Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Thu, 10 Dec 2020 15:41:42 -0700 Subject: [PATCH 09/11] still can't compile healpy --- .github/workflows/python-package.yml | 6 +- .travis.yml | 107 --------------------------- doc/desiInstall.rst | 62 ++++++++-------- py/desiutil/test/test_install.py | 30 ++++---- 4 files changed, 49 insertions(+), 156 deletions(-) delete mode 100644 .travis.yml diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8ab52e66..13666b31 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,7 +18,7 @@ jobs: matrix: os: [ubuntu-latest] python-version: [3.6, 3.7, 3.8] - astropy-version: ['<3.0', '<4.1'] #, '>4.0'] + # astropy-version: ['<3.0', '<4.1'] #, '<5.0'] steps: - name: Checkout code @@ -34,8 +34,8 @@ jobs: python -m pip install --upgrade pip wheel python -m pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi - pip install -U "astropy${{ matrix.astropy-version }}" - if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy==1.12.9"; fi +# pip install -U "astropy${{ matrix.astropy-version }}" +# if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy==1.12.9"; fi - name: Run the test run: | pytest diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6876f5a7..00000000 --- a/.travis.yml +++ /dev/null @@ -1,107 +0,0 @@ -# -# IMPORTANT. OS X support is so experimental that you have to specifically -# request to have it enabled, on a per-repository basis. See -# http://docs.travis-ci.com/user/multi-os/#Manual-intervention-required -# -# This file will still work, without OS X enabled, it just won't run those -# tests. -# -language: python - -os: - - linux - # - osx - -# xenial = Ubuntu 16.04 -# bionic = Ubuntu 18.04 -dist: xenial -# Setting sudo to false opts in to Travis-CI container-based builds. -sudo: false - -# The apt packages below are needed for sphinx builds, which can no longer -# be installed with sudo apt-get. -addons: - apt: - packages: - - graphviz - - texlive-latex-extra - - dvipng -python: - # - 3.5 - - 3.6 - - 3.7 - - 3.8 -env: - global: - # The following versions are the 'default' for tests, unless - # overidden underneath. They are defined here in order to save having - # to repeat them for all configurations. - - MAIN_CMD='pytest' - - SETUP_CMD='' - # - MAIN_CMD='python -m unittest' - # - SETUP_CMD='discover -t py -v desiutil.test' - matrix: - # - SETUP_CMD='egg_info' - # - SETUP_CMD='bdist_egg' - -matrix: - # Don't wait for allowed failures. - fast_finish: true - - # OS X support is still experimental, so don't penalize failures. - allow_failures: - - os: osx - - os: linux - python: 3.6 - env: MAIN_CMD='pycodestyle' SETUP_CMD='--count py/desiutil' - - include: - # Check for sphinx doc build warnings - we do this first because it - # runs for a long time - - os: linux - python: 3.6 - env: MAIN_CMD='sphinx-build' SETUP_CMD='-W --keep-going -b html doc doc/_build/html' - - # Coverage test, pass the results to coveralls. - - os: linux - python: 3.6 - env: SETUP_CMD='--cov' - - - os: linux - python: 3.6 - env: ASTROPY_VERSION='>=4.0' - - - os: linux - python: 3.6 - env: ASTROPY_VERSION='<4.0' - - # Experimental OS X test. - # As of March 2017, Python builds on OS X are not available. - # See https://docs.travis-ci.com/user/languages/python/ - # - os: osx - # python: 3.5 - # env: SETUP_CMD='test' - - # PEP 8 compliance. - - os: linux - python: 3.6 - env: MAIN_CMD='pycodestyle' SETUP_CMD='--count py/desiutil' - -# before_install: -# - curl ipinfo.io -# - python -c 'import setuptools; print(setuptools.__version__)' - -install: - - pip install -U pip - - if [[ "${MAIN_CMD}" == pycodestyle ]]; then pip install pycodestyle; fi - - if [[ "${MAIN_CMD}" == sphinx-build* ]]; then pip install Sphinx; fi - - if [[ "${MAIN_CMD}" == pytest ]]; then pip install -r requirements.txt; pip install . ; fi - - if [[ "${SETUP_CMD}" == '--cov' ]]; then pip install pytest-cov coveralls; fi - # - if [[ "${MAIN_CMD}" == python* || "${MAIN_CMD}" == coverage* ]]; then pip install -r requirements.txt; fi - - if [[ -n "${ASTROPY_VERSION}" ]]; then pip install -U "astropy${ASTROPY_VERSION}"; fi - -script: - - $MAIN_CMD $SETUP_CMD - -after_success: - - if [[ "${SETUP_CMD}" == '--cov' ]]; then coveralls; fi diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index a732b272..4860b7ab 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -35,9 +35,9 @@ Configuring desiInstall :command:`desiInstall` has many options, which are best viewed by typing ``desiInstall --help``. -In addition, desiInstall both reads and sets several environment variables. +In addition, :command:`desiInstall` both reads and sets several environment variables. -Environment variables that strongly affect the behavior of desiInstall. +Environment variables that strongly affect the behavior of :command:`desiInstall`. :envvar:`DESICONDA` This variable contains the path to the `DESI+Anaconda infrastructure`_. @@ -54,18 +54,18 @@ Environment variables that strongly affect the behavior of desiInstall. This variable is read to determine the username to pass to, *e.g.*, :command:`svn`. -Environment variables that are *set* by desiInstall for use by +Environment variables that are *set* by :command:`desiInstall` for use by :command:`python setup.py` or :command:`make`. :envvar:`INSTALL_DIR` - This variable is *set* by desiInstall to the directory that will contain + This variable is *set* by :command:`desiInstall` to the directory that will contain the final, installed version of the software package. :envvar:`PRODUCT_VERSION` - This variable is *set* by desiInstall, with ``PRODUCT`` replaced by the + This variable is *set* by :command:`desiInstall`, with ``PRODUCT`` replaced by the actual name of the software being installed, *e.g.*, :envvar:`DESISPEC_VERSION`. :envvar:`WORKING_DIR` - This variable is *set* by desiInstall to the path containing a downloaded, + This variable is *set* by :command:`desiInstall` to the path containing a downloaded, expanded software package. Environment variables related to the Modules infrastructure that may be @@ -99,24 +99,24 @@ manipulated by setting up Modules, or loading Module files. Directory Structure Assumed by the Install ========================================== -desiInstall is primarily intended to run in a production environment that +:command:`desiInstall` is primarily intended to run in a production environment that supports Module files, *i.e.* at NERSC. -*desiInstall does not install a Modules infrastructure for you.* You have to +*:command:`desiInstall` does not install a Modules infrastructure for you.* You have to do this yourself, if your system does not already have this. For the purposes of this section, we define ``$product_root`` as the -directory that desiInstall will be writing to. For standard NERSC installs it +directory that :command:`desiInstall` will be writing to. For standard NERSC installs it defaults to a pre-defined value. ``$product_root`` may contain the following directories: code/ This contains the installed code, the result of :command:`python setup.py install` or :command:`make install`. The code is always placed in a ``product/version`` - directory. So for example, the full path to desiInstall might be + directory. So for example, the full path to :command:`desiInstall` might be ``$product_root/code/desiutil/1.8.0/bin/desiInstall``. modulefiles/ - This contains the the Module files installed by desiInstall. A Module + This contains the the Module files installed by :command:`desiInstall`. A Module file is almost always named ``product/version``. For example, the Module file for desiutil might be ``$product_root/modulefiles/desiutil/1.8.0``. @@ -157,17 +157,17 @@ Stages of the Install Input Validation ---------------- -desiInstall checks the command-line input, verifying that the user has +:command:`desiInstall` checks the command-line input, verifying that the user has specified a product and a version to install. Product/Version Parsing ----------------------- Because of the structures of the DESI code repositories, it is sometimes necessary -to specify a directory name along with the product name. desiInstall contains -a list of known products, but it is not necessarily complete. desiInstall parses +to specify a directory name along with the product name. :command:`desiInstall` contains +a list of known products, but it is not necessarily complete. :command:`desiInstall` parses the input to determine the base name and base version to install. At this -stage desiInstall also determines whether a branch install [#]_ has +stage :command:`desiInstall` also determines whether a branch install [#]_ has been requested. The internal list of known products can be added to or overridden on the @@ -187,7 +187,7 @@ matters to the product actually being installed. Product Existence ----------------- -After the product name and version have been determined, desiInstall +After the product name and version have been determined, :command:`desiInstall` constructs the full URL pointing to the product/version and runs the code necessary to verify that the product/version really exists. Typically, this will be :command:`svn ls`, unless a GitHub install is detected. @@ -196,9 +196,9 @@ Download Code ------------- The code is downloaded, using :command:`svn export` for standard (tag) installs, or -:command:`svn checkout` for branch installs. For GitHub installs, desiInstall +:command:`svn checkout` for branch installs. For GitHub installs, :command:`desiInstall` will look for a release tarball, or do a :command:`git clone` for tag or branch -installs. desiInstall will set the environment variable :envvar:`WORKING_DIR` +installs. :command:`desiInstall` will set the environment variable :envvar:`WORKING_DIR` to point to the directory containing this downloaded code. Determine Build Type @@ -211,15 +211,15 @@ plain This is the default build type. With this build type, the downloaded code is simply copied to the final install directory. py - If a setup.py file is detected, desiInstall will attempt to execute + If a setup.py file is detected, :command:`desiInstall` will attempt to execute :command:`python setup.py install`. This build type can be suppressed with the command line option ``--compile-c``. make - If a Makefile is detected, desiInstall will attempt to execute + If a Makefile is detected, :command:`desiInstall` will attempt to execute :command:`make install`. src If a Makefile is not present, but a src/ directory is, - desiInstall will attempt to execute :command:`make -C src all`. This build type + :command:`desiInstall` will attempt to execute :command:`make -C src all`. This build type *is* mutually exclusive with 'make', but is not mutually exclusive with the other types. @@ -238,10 +238,10 @@ predetermined based on the value of :envvar:`NERSC_HOST`:: The actual install directory is determined by appending ``/code/product/verson`` to the combining the top-level directory listed above. -If the install directory already exists, desiInstall will exit, unless the +If the install directory already exists, :command:`desiInstall` will exit, unless the ``--force`` parameter is supplied on the command line. -desiInstall will set the environment variable :envvar:`INSTALL_DIR` to point to the +:command:`desiInstall` will set the environment variable :envvar:`INSTALL_DIR` to point to the install directory. .. _`DESI+Anaconda infrastructure`: https://github.com/desihub/desiconda @@ -250,27 +250,27 @@ install directory. Module Infrastructure --------------------- -desiInstall sets up the Modules infrastructure by running code in +:command:`desiInstall` sets up the Modules infrastructure by running code in :mod:`desiutil.modules` that is *based on* the Python init file supplied by the Modules infrastructure, but updated to be both Python 2 and Python 3 compatible. Find Module File ---------------- -desiInstall will search for a module file in ``$WORKING_DIR/etc``. If that -module file is not found, desiInstall will use the file that comes with +:command:`desiInstall` will search for a module file in ``$WORKING_DIR/etc``. If that +module file is not found, :command:`desiInstall` will use the file that comes with desiutil_ (*i.e.*, this product's own module file). Load Dependencies ----------------- -desiInstall will scan the module file identified in the previous stage, and +:command:`desiInstall` will scan the module file identified in the previous stage, and will module load any dependencies found in the file. Configure Module File --------------------- -desiInstall will scan :envvar:`WORKING_DIR` to determine the details that need +:command:`desiInstall` will scan :envvar:`WORKING_DIR` to determine the details that need to be added to the module file. The final module file will then be written into the DESI module directory at NERSC. If ``--default`` is specified on the command line, an appropriate .version file will be created. @@ -278,7 +278,7 @@ on the command line, an appropriate .version file will be created. Load Module ----------- -desiInstall will load the module file just created to set up any environment +:command:`desiInstall` will load the module file just created to set up any environment variables needed by the install. At this point it is also safe to assume that the environment variables :envvar:`WORKING_DIR` and :envvar:`INSTALL_DIR` exist. It will also set :envvar:`PRODUCT_VERSION`, where ``PRODUCT`` will be replaced @@ -314,11 +314,11 @@ will be run in :envvar:`INSTALL_DIR`. Download Extra Data ------------------- -If desiInstall detects ``etc/product_data.sh``, where ``product`` should be +If :command:`desiInstall` detects ``etc/product_data.sh``, where ``product`` should be replaced by the actual name of the package, it will download extra data not bundled with the code. The script should download data *directly* to :envvar:`INSTALL_DIR`. The script should *only* be used -with desiInstall and Travis tests. Note that here are other, better ways to +with :command:`desiInstall` and Travis tests. Note that here are other, better ways to install and manipulate data that is bundled *with* a Python package. Fix Permissions diff --git a/py/desiutil/test/test_install.py b/py/desiutil/test/test_install.py index ae0642b8..41ede613 100644 --- a/py/desiutil/test/test_install.py +++ b/py/desiutil/test/test_install.py @@ -169,7 +169,7 @@ def test_identify_branch(self): self.assertEqual(url, ('https://github.com/desihub/desiutil/archive/' + '1.0.0.tar.gz')) - options = self.desiInstall.get_options(['desiutil', 'branches/master']) + options = self.desiInstall.get_options(['desiutil', 'branches/main']) out = self.desiInstall.get_product_version() url = self.desiInstall.identify_branch() self.assertEqual(url, @@ -270,7 +270,7 @@ def test_anaconda_version(self): def test_default_nersc_dir(self): """Test determination of the NERSC installation root. """ - options = self.desiInstall.get_options(['desiutil', 'master']) + options = self.desiInstall.get_options(['desiutil', 'branches/main']) self.desiInstall.nersc = 'edison' nersc_dir = self.desiInstall.default_nersc_dir() edison_nersc_dir = '/global/common/software/desi/edison/desiconda/current' @@ -279,7 +279,7 @@ def test_default_nersc_dir(self): self.assertEqual(nersc_dir, edison_nersc_dir) options = self.desiInstall.get_options(['--anaconda', 'frobulate', - 'desiutil', 'master']) + 'desiutil', '1.2.3']) self.desiInstall.nersc = 'datatran' nersc_dir = self.desiInstall.default_nersc_dir() self.assertEqual(nersc_dir, '/global/common/software/desi/datatran/desiconda/frobulate') @@ -291,35 +291,35 @@ def test_set_install_dir(self): del environ['NERSC_HOST'] options = self.desiInstall.get_options(['--root', '/fake/root/directory', - 'desiutil', 'master']) + 'desiutil', '1.2.3']) with self.assertRaises(DesiInstallException): install_dir = self.desiInstall.set_install_dir() options = self.desiInstall.get_options(['--root', self.data_dir, - 'desiutil', 'master']) + 'desiutil', '1.2.3']) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() self.assertEqual(install_dir, join(self.data_dir, 'code', 'desiutil', - 'master')) + '1.2.3')) # Test for presence of existing directory. tmpdir = join(self.data_dir, 'code') mkdir(tmpdir) mkdir(join(tmpdir, 'desiutil')) - mkdir(join(tmpdir, 'desiutil', 'master')) + mkdir(join(tmpdir, 'desiutil', 'main')) options = self.desiInstall.get_options(['--root', self.data_dir, - 'desiutil', 'master']) + 'desiutil', 'branches/main']) self.desiInstall.get_product_version() with self.assertRaises(DesiInstallException) as cm: install_dir = self.desiInstall.set_install_dir() self.assertEqual(str(cm.exception), "Install directory, {0}, already exists!".format( - join(tmpdir, 'desiutil', 'master'))) + join(tmpdir, 'desiutil', 'main'))) options = self.desiInstall.get_options(['--root', self.data_dir, '--force', 'desiutil', - 'master']) + 'branches/main']) self.assertTrue(self.desiInstall.options.force) self.desiInstall.get_product_version() install_dir = self.desiInstall.set_install_dir() - self.assertFalse(isdir(join(tmpdir, 'desiutil', 'master'))) + self.assertFalse(isdir(join(tmpdir, 'desiutil', 'branches/main'))) if isdir(tmpdir): rmtree(tmpdir) # Test NERSC installs. Unset DESI_PRODUCT_ROOT for this to work. @@ -340,13 +340,13 @@ def test_start_modules(self): """ options = self.desiInstall.get_options(['-m', '/fake/modules/directory', - 'desiutil', 'master']) + 'desiutil', 'branches/main']) with self.assertRaises(DesiInstallException) as cm: status = self.desiInstall.start_modules() self.assertEqual(str(cm.exception), ("Could not initialize Modules " + "with MODULESHOME={0}!").format( '/fake/modules/directory')) - options = self.desiInstall.get_options(['desiutil', 'master']) + options = self.desiInstall.get_options(['desiutil', 'branches/main']) self.assertEqual(options.moduleshome, environ['MODULESHOME']) status = self.desiInstall.start_modules() self.assertTrue(callable(self.desiInstall.module)) @@ -373,9 +373,9 @@ def test_nersc_module_dir(self): def test_cleanup(self): """Test the cleanup stage of the install. """ - options = self.desiInstall.get_options(['desiutil', 'master']) + options = self.desiInstall.get_options(['desiutil', 'branches/main']) self.desiInstall.original_dir = getcwd() - self.desiInstall.working_dir = join(self.data_dir, 'desiutil-master') + self.desiInstall.working_dir = join(self.data_dir, 'desiutil') mkdir(self.desiInstall.working_dir) chdir(self.desiInstall.working_dir) self.desiInstall.cleanup() From a9bc08850a82642910d91498765706fe58266473 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 11 Dec 2020 13:49:19 -0700 Subject: [PATCH 10/11] allow "main" abbreviation; shorten test names --- .github/workflows/python-package.yml | 33 ++++++++++------------------ doc/desiInstall.rst | 4 +++- py/desiutil/install.py | 3 ++- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 13666b31..8ec519e8 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -11,7 +11,7 @@ on: jobs: tests: - name: Standard Python tests. + name: Unit tests runs-on: ${{ matrix.os }} strategy: fail-fast: true @@ -37,11 +37,10 @@ jobs: # pip install -U "astropy${{ matrix.astropy-version }}" # if [ "${{ matrix.astropy-version }}" = "<3.0" ]; then pip install -U "healpy==1.12.9"; fi - name: Run the test - run: | - pytest + run: pytest coverage: - name: Code coverage test. + name: Test coverage runs-on: ${{ matrix.os }} strategy: fail-fast: true @@ -64,16 +63,14 @@ jobs: python -m pip install pytest pytest-cov coveralls if [ -f requirements.txt ]; then pip install -r requirements.txt; pip install .; fi - name: Run the test with coverage - run: | - pytest --cov + run: pytest --cov - name: Coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - coveralls + run: coveralls docs: - name: Check Sphinx documentation. + name: Doc test runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -91,15 +88,12 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install Python dependencies - run: | - python -m pip install --upgrade pip wheel - python -m pip install Sphinx + run: python -m pip install --upgrade pip wheel Sphinx - name: Test the documentation - run: | - sphinx-build -W --keep-going -b html doc doc/_build/html + run: sphinx-build -W --keep-going -b html doc doc/_build/html style: - name: Check code style; allow failures. + name: Style check runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -117,11 +111,8 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install Python dependencies - run: | - python -m pip install --upgrade pip wheel - python -m pip install pycodestyle - - name: Test the style + run: python -m pip install --upgrade pip wheel pycodestyle + - name: Test the style; failures are allowed # This is equivalent to an allowed falure. continue-on-error: true - run: | - pycodestyle --count py/desiutil + run: pycodestyle --count py/desiutil diff --git a/doc/desiInstall.rst b/doc/desiInstall.rst index 4860b7ab..714300a4 100644 --- a/doc/desiInstall.rst +++ b/doc/desiInstall.rst @@ -27,7 +27,9 @@ by prepending ``branches/`` to the version name:: desiInstall desiutil branches/branch-compile -Finally, for Subversion repositories, ``trunk`` is a shorthand for ``branches/trunk``. +Finally, for Subversion repositories, ``trunk`` is a shorthand for ``branches/trunk``; +for GitHub repositories, ``main`` is a shorthand for ``branches/main``. All +other branch names *must* be prepended with ``branches/``. Configuring desiInstall ======================= diff --git a/py/desiutil/install.py b/py/desiutil/install.py index 047bfa99..af15f1c5 100644 --- a/py/desiutil/install.py +++ b/py/desiutil/install.py @@ -311,7 +311,8 @@ def identify_branch(self): The full path to the branch code. """ self.is_branch = (self.options.product_version.startswith('branches') or - self.options.product_version == 'trunk') + self.options.product_version == 'trunk' or + self.options.product_version == 'main') if self.is_branch: if self.github: self.product_url = self.fullproduct + '.git' From 6df17aa3bb98c38e943d2642670386432ba030d6 Mon Sep 17 00:00:00 2001 From: Benjamin Alan Weaver Date: Fri, 11 Dec 2020 14:06:38 -0700 Subject: [PATCH 11/11] update changes.rst --- doc/changes.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/changes.rst b/doc/changes.rst index 77053e5a..b35150e1 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -2,15 +2,18 @@ Change Log ========== -3.0.4 (unreleased) +3.1.0 (2020-12-11) ------------------ -* Add :class:`~desiutil.timer.Timer` class to standardize timing reports (PRs `#151`_, `#152`_). +* Migrate unit tests to GitHub Actions; allow :command:`desiInstall` to handle a + diversity of possible branch names (PR `#156`_). * Add :mod:`~desiutil.redirect` for utilites related to redirecting STDOUT (PR `#153`_). +* Add :class:`~desiutil.timer.Timer` class to standardize timing reports (PRs `#151`_, `#152`_). .. _`#151`: https://github.com/desihub/desiutil/pull/151 .. _`#152`: https://github.com/desihub/desiutil/pull/152 .. _`#153`: https://github.com/desihub/desiutil/pull/153 +.. _`#156`: https://github.com/desihub/desiutil/pull/156 3.0.3 (2020-08-04) ------------------