Merge pull request #35 from DiamondLightSource/test-lockfiles

Multiple fixes.

.containerignore → .dockerignore

@@ -1,4 +1,3 @@
-Dockerfile
 build/
 dist/
 .mypy_cache

.github/pages/make_switcher.py

@@ -59,7 +59,7 @@ def get_versions(ref: str, add: Optional[str], remove: Optional[str]) -> List[st
 def write_json(path: Path, repository: str, versions: str):
     org, repo_name = repository.split("/")
     struct = [
-        dict(name=version, url=f"https://{org}.github.io/{repo_name}/{version}/")
+        dict(version=version, url=f"https://{org}.github.io/{repo_name}/{version}/")
         for version in versions
     ]
     text = json.dumps(struct, indent=2)

.github/workflows/code.yml

@@ -115,6 +115,7 @@ jobs:
         run: |
           docker run --name test build bash /project/.github/workflows/container_tests.sh
           docker cp test:/project/dist .
+          docker cp test:/project/lockfiles .
           docker cp test:/project/cov.xml .
 
       - name: Upload coverage to Codecov
@@ -131,30 +132,27 @@ jobs:
           context: .
           labels: ${{ steps.meta.outputs.labels }}
 
-      - name: Check runtime
+      - name: Test cli works in runtime image
         # check that the first tag can run with --version parameter
-        run: docker run $(echo ${{ steps.meta.outputs.tags }} | sed -e 's/\s.*$//') --version
+        run: docker run $(echo ${{ steps.meta.outputs.tags }} | head -1) --version
+
+      - name: Test cli works in sdist installed in local python
+        # ${GITHUB_REPOSITORY##*/} is the repo name without org
+        # Replace this with the cli command if different to the repo name
+        # (python3-pip-skeleton-cli replaces this with python3-pip-skeleton)
+        run: pip install dist/*.gz && python3-pip-skeleton --version
 
       - name: Upload build files
         uses: actions/upload-artifact@v3
         with:
           name: dist
           path: dist
 
-  sdist:
-    needs: container
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/download-artifact@v3
-
-      - name: Install sdist in a venv and check cli works
-        # ${GITHUB_REPOSITORY##*/} is the repo name without org
-        # Replace this with the cli command if different to the repo name
-        # (In the python3-pip-skeleton-cli this is already renamed)
-        run: |
-          pip install dist/*.gz
-          python3-pip-skeleton --version
+      - name: Upload lock files
+        uses: actions/upload-artifact@v3
+        with:
+          name: lockfiles
+          path: lockfiles
 
   release:
     # upload to PyPI and make a release on every tag
@@ -171,13 +169,14 @@ jobs:
         uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5 # v0.1.14
         with:
           prerelease: ${{ contains(github.ref_name, 'a') || contains(github.ref_name, 'b') || contains(github.ref_name, 'rc') }}
-          files: dist/*
+          files: |
+            dist/
+            lockfiles/
           generate_release_notes: true
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Publish to PyPI
-        env:
-          TWINE_USERNAME: __token__
-          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        run: pipx run twine upload dist/*.whl dist/*.tar.gz
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/container_tests.sh

@@ -6,7 +6,8 @@ source /venv/bin/activate
 
 touch requirements_dev.txt
 pip install -r requirements_dev.txt -e .[dev]
-pip freeze --exclude-editable > dist/requirements_dev.txt
+mkdir -p lockfiles
+pip freeze --exclude-editable > lockfiles/requirements_dev.txt
 
 pipdeptree
 

.github/workflows/docs.yml

@@ -41,12 +41,14 @@ jobs:
       - name: Build docs
         run: tox -e docs
 
+      - name: Sanitize ref name for docs version
+        run: echo "DOCS_VERSION=${GITHUB_REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV
+
       - name: Move to versioned directory
-        # e.g. main or 0.1.2
-        run: mv build/html ".github/pages/${{ github.ref_name }}"
+        run: mv build/html .github/pages/$DOCS_VERSION
 
       - name: Write switcher.json
-        run: python .github/pages/make_switcher.py --add "${{ github.ref_name }}" ${{ github.repository }} .github/pages/switcher.json
+        run: python .github/pages/make_switcher.py --add $DOCS_VERSION ${{ github.repository }} .github/pages/switcher.json
 
       - name: Publish Docs to gh-pages
         if: github.event_name == 'push'

.github/workflows/docs_clean.yml

@@ -24,17 +24,20 @@ jobs:
 
       - name: removing documentation for branch ${{ github.event.ref }}
         if: ${{ github.event_name != 'workflow_dispatch' }}
-        run: echo "remove_me=${{ github.event.ref }}" >> $GITHUB_ENV
+        run: echo "REF_NAME=${{ github.event.ref }}" >> $GITHUB_ENV
 
       - name: manually removing documentation version ${{ github.event.inputs.version }}
         if: ${{ github.event_name == 'workflow_dispatch' }}
-        run: echo "remove_me=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+        run: echo "REF_NAME=${{ github.event.inputs.version }}" >> $GITHUB_ENV
+
+      - name: Sanitize ref name for docs version
+        run: echo "DOCS_VERSION=${REF_NAME//[^A-Za-z0-9._-]/_}" >> $GITHUB_ENV
 
       - name: update index and push changes
         run: |
-          rm -r ${{ env.remove_me }}
-          python make_switcher.py --remove ${{ env.remove_me }} ${{ github.repository }} switcher.json
+          rm -r ${{ env.DOCS_VERSION }}
+          python make_switcher.py --remove $DOCS_VERSION ${{ github.repository }} switcher.json
           git config --global user.name 'GitHub Actions Docs Cleanup CI'
           git config --global user.email '[email protected]'
-          git commit -am"removing redundant docs version ${{ env.remove_me }}"
+          git commit -am "Removing redundant docs version $DOCS_VERSION"
           git push

.gitignore

@@ -64,4 +64,6 @@ target/
 .venv*
 venv*
 
+# further build artifacts
+lockfiles/
 

Dockerfile

@@ -18,29 +18,34 @@ RUN apt-get update && apt-get upgrade -y && \
     && busybox --install
 
 COPY . /project
+WORKDIR /project
 
-RUN cd /project && \
-    pip install --upgrade pip build && \
+# make the wheel outside of the venv so 'build' does not dirty requirements.txt
+RUN pip install --upgrade pip build && \
     export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) && \
-    python -m build --sdist --wheel && \
+    git diff && \
+    python -m build && \
     touch requirements.txt
 
+# set up a virtual environment and put it in PATH
 RUN python -m venv /venv
 ENV PATH=/venv/bin:$PATH
 ENV TOX_DIRECT=1
 
-RUN cd /project && \
-    pip install --upgrade pip && \
+# install the wheel and generate the requirements file
+RUN pip install --upgrade pip && \
     pip install -r requirements.txt dist/*.whl && \
-    pip freeze  > dist/requirements.txt && \
+    mkdir -p lockfiles && \
+    pip freeze  > lockfiles/requirements.txt && \
     # we don't want to include our own wheel in requirements - remove with sed
     # and replace with a comment to avoid a zero length asset upload later
-    sed -i '/file:/s/^/# Requirements for /' dist/requirements.txt
+    sed -i '/file:/s/^/# Requirements for /' lockfiles/requirements.txt
 
 FROM python:3.10-slim as runtime
 
 # Add apt-get system dependecies for runtime here if needed
 
+# copy the virtual environment from the build stage and put it in PATH
 COPY --from=build /venv/ /venv/
 ENV PATH=/venv/bin:$PATH
 

README.rst

@@ -26,9 +26,9 @@ It integrates the following tools:
   - which verifies all the things that CI does
 - If you use VSCode, it will run black, flake8, isort and mypy on save
 
-The the related skeleton_ repo for this module contains the source 
-code that can be merged into new or existing projects, and pulled from to 
-keep them up to date. It can also serve as a working example for those who 
+The the related skeleton_ repo for this module contains the source
+code that can be merged into new or existing projects, and pulled from to
+keep them up to date. It can also serve as a working example for those who
 would prefer to cherry-pick.
 
 .. _skeleton: https://github.com/DiamondLightSource/python3-pip-skeleton
@@ -43,12 +43,12 @@ and existing projects::
 
     python3-pip-skeleton existing /path/to/existing/repo --org my_github_user_or_org
 
-.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/workflows/Code%20CI/badge.svg?branch=main
-    :target: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/actions?query=workflow%3A%22Code+CI%22
+.. |code_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/actions/workflows/code.yml/badge.svg?branch=main
+    :target: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/actions/workflows/code.yml
     :alt: Code CI
 
-.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/workflows/Docs%20CI/badge.svg?branch=main
-    :target: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/actions?query=workflow%3A%22Docs+CI%22
+.. |docs_ci| image:: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/actions/workflows/docs.yml/badge.svg?branch=main
+    :target: https://github.com/DiamondLightSource/python3-pip-skeleton-cli/actions/workflows/docs.yml
     :alt: Docs CI
 
 .. |coverage| image:: https://codecov.io/gh/DiamondLightSource/python3-pip-skeleton-cli/branch/main/graph/badge.svg

docs/conf.py

@@ -24,7 +24,6 @@
     git_branch = check_output("git branch --show-current".split(), cwd=root)
     version = git_branch.decode().strip()
 else:
-    branch = "main"
     version = release
 
 extensions = [

docs/developer/how-to/build-docs.rst

@@ -17,4 +17,22 @@ locally with a web browse::
 
     $ firefox build/html/index.html
 
+Autobuild
+---------
+
+You can also run an autobuild process, which will watch your ``docs``
+directory for changes and rebuild whenever it sees changes, reloading any
+browsers watching the pages::
+
+    $ tox -e docs autobuild
+
+You can view the pages at localhost::
+
+    $ firefox http://localhost:8000
+
+If you are making changes to source code too, you can tell it to watch
+changes in this directory too::
+
+    $ tox -e docs autobuild -- --watch src
+
 .. _sphinx: https://www.sphinx-doc.org/

setup.cfg

@@ -32,15 +32,16 @@ setup_requires =
 [options.extras_require]
 # For development tests/docs
 dev =
-    black==22.8.0
+    black==22.10.0
     flake8-isort
     isort>5.0
     mypy
     pipdeptree
     pre-commit
-    pydata-sphinx-theme
+    pydata-sphinx-theme < 0.10.1
     pytest-cov
     setuptools_scm[toml]>=6.2
+    sphinx-autobuild
     sphinx-copybutton
     sphinx-design
     tox
@@ -122,5 +123,7 @@ allowlist_externals = pre-commit
 commands = pre-commit run --all-files {posargs}
 
 [testenv:docs]
-allowlist_externals = sphinx-build
-commands = sphinx-build -EWT --keep-going docs build/html {posargs}
+allowlist_externals =
+    sphinx-build
+    sphinx-autobuild
+commands = sphinx-{posargs:build -EW --keep-going} -T docs build/html