From 9948708a6ab7242259530719f25e264c939f3c46 Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 20 Jun 2024 17:37:14 +0100 Subject: [PATCH 1/9] try to run ci in docker --- .github/workflows/tests.yml | 70 ++++++++++++++++++++----------------- Dockerfile | 51 +++++++++++++-------------- Makefile | 4 +++ tests/update_source_repo.py | 2 +- 4 files changed, 67 insertions(+), 60 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d9fdc8766..fe4e01f6c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,8 @@ jobs: steps: - uses: actions/checkout@v4 + # submodules: recursive + # fetch-depth: 0 - name: checkout submodules shell: bash @@ -51,46 +53,50 @@ jobs: git submodule init git submodule status | cut -d" " -f2 | xargs -n1 -P0 git submodule update - - name: setup Git - shell: bash + - name: warm up docker image run: | - git config --global user.email "elspeth@example.com" - git config --global user.name "Elspeth See-Eye" - git config --global init.defaultBranch main - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: 3.12 + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "echo hello world" - - name: Install apt stuff and other dependencies - shell: bash + - name: uv pip install --upgrade systemwide run: | - sudo add-apt-repository ppa:mozillateam/ppa - sudo apt update -y - sudo apt install -y \ - asciidoctor \ - language-pack-en \ - ruby-coderay \ - ruby-pygments.rb \ - firefox-esr \ - tree - pip install uv + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "uv pip install --system --upgrade ." - - name: Install Python requirements.txt globally - shell: bash + - name: uv create virtualenv in mounted /app folder at .venv run: | - uv pip install --system . + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "uv venv && uv pip install --upgrade ." - - name: Install Python requirements.txt into virtualenv - shell: bash + - name: git mark dirs safe (due to volume mount perms issue) run: | - make .venv/bin - - - name: Run chapter test - shell: bash + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "git config --global --add safe.directory '*'" + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "cat ~/.gitconfig" + + - name: run chapter tests run: | - make ${{ matrix.test_chapter }} + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "make ${{ matrix.test_chapter }}" - name: Save tempdir path to an env var if: always() diff --git a/Dockerfile b/Dockerfile index e2644cf28..5e90a9fa7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,40 +1,37 @@ FROM python:slim +# -- Dockerfile is intended mainly for use in CI -- + +RUN apt-get update -y && \ + apt-get install -y \ + asciidoctor \ + curl \ + firefox-esr \ + git \ + locales \ + make \ + ruby-pygments.rb \ + tree \ + wget \ + && rm -rf /var/lib/apt/lists/* + +ENV LC_ALL=en_GB.UTF-8 LANG=en_GB.UTF8 LANGUAGE=en_GB:en +RUN echo "en_GB.UTF-8 UTF-8" > /etc/locale.gen +RUN locale-gen -# -- WIP -- -# this dockerfile is a work in progress, -# the vague intention is to use it for CI. - -# RUN add-apt-repository ppa:mozillateam/ppa && \ -RUN apt-get update -y - -RUN apt-get install -y \ - git \ - asciidoctor \ - # language-pack-en \ - ruby-pygments.rb \ - firefox-esr \ - tree \ - locales - -RUN apt-get install -y \ - make \ - curl - -RUN locale-gen en_GB.UTF-8 -# RUN pip install uv ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh -RUN /install.sh && rm /install.sh -RUN ln -s $HOME/.cargo/bin/uv /usr/bin/uv +RUN /install.sh && rm /install.sh && \ + ln -s $HOME/.cargo/bin/uv /usr/bin/uv RUN git config --global user.email "elspeth@example.com" && \ git config --global user.name "Elspeth See-Eye" && \ git config --global init.defaultBranch main WORKDIR /app -RUN uv venv .venv - COPY pyproject.toml pyproject.toml -RUN uv pip install . +# install python deps systemwide with uv +ENV PIP_CACHE_DIR=/var/cache/pip + +RUN uv pip install --system . CMD bash diff --git a/Makefile b/Makefile index a833923cd..147f5cb5f 100644 --- a/Makefile +++ b/Makefile @@ -167,3 +167,7 @@ unit-test: chapter_01.html $(VENV)/bin clean: rm -rf $(TMPDIR) rm -v $(HTML_PAGES) + +.PHONY: docker-build +docker-build: + docker build --platform=linux/amd64 -t hjwp/obeythetestinggoat-book-tester:latest . diff --git a/tests/update_source_repo.py b/tests/update_source_repo.py index 09d92c3ab..2b4bf5780 100755 --- a/tests/update_source_repo.py +++ b/tests/update_source_repo.py @@ -49,7 +49,7 @@ def update_sources_for_chapter(chapter, previous_chapter=None): # make sure branch for previous chapter is available to start tests subprocess.check_output(["git", "checkout", previous_chapter], cwd=source_dir) subprocess.check_output( - ["git", "reset", "--hard", "{}/{}".format(REMOTE, previous_chapter)], + ["git", "reset", "--hard", f"{REMOTE}/{previous_chapter}"], cwd=source_dir, ) From f2cef716f72f8c172519514e7fb68c8d9fb96891 Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 25 Jun 2024 18:36:52 +0100 Subject: [PATCH 2/9] give up on UK english git --- chapter_01.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chapter_01.asciidoc b/chapter_01.asciidoc index 72e8c01f8..9c658d38e 100644 --- a/chapter_01.asciidoc +++ b/chapter_01.asciidoc @@ -298,7 +298,7 @@ $ *ls* db.sqlite3 functional_tests.py manage.py superlists $ *git init .* -Initialised empty Git repository in ...goat-book/.git/ +Initialized empty Git repository in ...goat-book/.git/ ---- // CSANAD: maybe we could mention that Git also provides hints about the default branch name From cb2db2664afc41167de6179f21f4be6ebe5fa08b Mon Sep 17 00:00:00 2001 From: Harry Date: Tue, 25 Jun 2024 18:57:45 +0100 Subject: [PATCH 3/9] OK apparently debian tree overcounts dirs just like macos --- chapter_08_prettification.asciidoc | 2 +- chapter_16_javascript.asciidoc | 2 +- chapter_CI.asciidoc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chapter_08_prettification.asciidoc b/chapter_08_prettification.asciidoc index 1d82e1ab1..b75e2a953 100644 --- a/chapter_08_prettification.asciidoc +++ b/chapter_08_prettification.asciidoc @@ -1062,7 +1062,7 @@ static/ ├── [...] └── bootstrap.min.js.map -16 directories, 169 files +17 directories, 169 files ---- `collectstatic` has also picked up all the CSS for the admin site. diff --git a/chapter_16_javascript.asciidoc b/chapter_16_javascript.asciidoc index 260239acf..792662972 100644 --- a/chapter_16_javascript.asciidoc +++ b/chapter_16_javascript.asciidoc @@ -419,7 +419,7 @@ src/lists/static/tests ├── jasmine.js └── jasmine_favicon.png -2 directories, 9 files +3 directories, 9 files ---- We need to go edit the _SpecRunner.html_ file diff --git a/chapter_CI.asciidoc b/chapter_CI.asciidoc index c8357283a..9103f9988 100644 --- a/chapter_CI.asciidoc +++ b/chapter_CI.asciidoc @@ -676,7 +676,7 @@ lists/static/tests/ ├── runner.js └── tests.html -0 directories, 4 files +1 directories, 4 files ---- Let's try it out: From a0c7a1a794d04acb503b02464d6ac1a6da591140 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 26 Jun 2024 10:48:18 +0100 Subject: [PATCH 4/9] give up on locale --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e90a9fa7..5cab04158 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,17 +7,12 @@ RUN apt-get update -y && \ curl \ firefox-esr \ git \ - locales \ make \ ruby-pygments.rb \ tree \ wget \ && rm -rf /var/lib/apt/lists/* -ENV LC_ALL=en_GB.UTF-8 LANG=en_GB.UTF8 LANGUAGE=en_GB:en -RUN echo "en_GB.UTF-8 UTF-8" > /etc/locale.gen -RUN locale-gen - ADD --chmod=755 https://astral.sh/uv/install.sh /install.sh RUN /install.sh && rm /install.sh && \ ln -s $HOME/.cargo/bin/uv /usr/bin/uv From 6ed85d7cc098aa432e0b26b68075eb4efb7fc38b Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 26 Jun 2024 11:03:25 +0100 Subject: [PATCH 5/9] add no-install-recommends and save 300mb --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5cab04158..e8aaf745c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM python:slim # -- Dockerfile is intended mainly for use in CI -- RUN apt-get update -y && \ - apt-get install -y \ + apt-get install -y --no-install-recommends \ asciidoctor \ curl \ firefox-esr \ From 043daa158778fecb2807a4bd14d0f788a5e9e8df Mon Sep 17 00:00:00 2001 From: Harry Date: Thu, 11 Jul 2024 10:06:01 +0100 Subject: [PATCH 6/9] no value in upgrading system deps, they are not persisted --- .github/workflows/tests.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fe4e01f6c..133e3c770 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,15 +61,7 @@ jobs: -t hjwp/obeythetestinggoat-book-tester:latest \ bash -c "echo hello world" - - name: uv pip install --upgrade systemwide - run: | - docker run --rm \ - --mount type=volume,source=rootmount,target=/root \ - --mount type=bind,source=./,target=/app \ - -t hjwp/obeythetestinggoat-book-tester:latest \ - bash -c "uv pip install --system --upgrade ." - - - name: uv create virtualenv in mounted /app folder at .venv + - name: uv create virtualenv in mounted /app folder at .venv and install latest/upgraded deps run: | docker run --rm \ --mount type=volume,source=rootmount,target=/root \ From 3677e45762c445302d9cfb6f132833a4246e1388 Mon Sep 17 00:00:00 2001 From: Harry Date: Fri, 2 Aug 2024 12:47:46 +0100 Subject: [PATCH 7/9] wip making some docker runner commands --- .github/workflows/tests.yml | 2 ++ Makefile | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 133e3c770..141dcbc65 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,6 +62,7 @@ jobs: bash -c "echo hello world" - name: uv create virtualenv in mounted /app folder at .venv and install latest/upgraded deps + # todo, replace with make docker-venv? run: | docker run --rm \ --mount type=volume,source=rootmount,target=/root \ @@ -83,6 +84,7 @@ jobs: bash -c "cat ~/.gitconfig" - name: run chapter tests + # todo, replace with make docker_${{ matrix.test_chapter }} ? run: | docker run --rm \ --mount type=volume,source=rootmount,target=/root \ diff --git a/Makefile b/Makefile index 147f5cb5f..894e47a50 100644 --- a/Makefile +++ b/Makefile @@ -72,6 +72,24 @@ testall4: build test_%: %.html $(TMPDIR) $(VENV)/bin/pytest -s --tb=short ./tests/$@.py +.PHONY: docker-venv +docker-venv: + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + --mount type=volume,source=venvmount,target=/app/.venv \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "uv venv .venv && uv pip install --upgrade ." + +.PHONY: test_% +docker_test_%: docker-venv %.html $(TMPDIR) + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + --mount type=volume,source=venvmount,target=/app/.venv \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "make $(subst docker_,,$@)" + # exhaustively list for nice tab-completion # .PHONY: test_chapter_01 From cc0a6770953400f9a2fbf14a3409581814b40360 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 7 Aug 2024 12:59:58 +0100 Subject: [PATCH 8/9] fix makefile commands --- Makefile | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 894e47a50..9e9033507 100644 --- a/Makefile +++ b/Makefile @@ -72,24 +72,6 @@ testall4: build test_%: %.html $(TMPDIR) $(VENV)/bin/pytest -s --tb=short ./tests/$@.py -.PHONY: docker-venv -docker-venv: - docker run --rm \ - --mount type=volume,source=rootmount,target=/root \ - --mount type=bind,source=./,target=/app \ - --mount type=volume,source=venvmount,target=/app/.venv \ - -t hjwp/obeythetestinggoat-book-tester:latest \ - bash -c "uv venv .venv && uv pip install --upgrade ." - -.PHONY: test_% -docker_test_%: docker-venv %.html $(TMPDIR) - docker run --rm \ - --mount type=volume,source=rootmount,target=/root \ - --mount type=bind,source=./,target=/app \ - --mount type=volume,source=venvmount,target=/app/.venv \ - -t hjwp/obeythetestinggoat-book-tester:latest \ - bash -c "make $(subst docker_,,$@)" - # exhaustively list for nice tab-completion # .PHONY: test_chapter_01 @@ -189,3 +171,26 @@ clean: .PHONY: docker-build docker-build: docker build --platform=linux/amd64 -t hjwp/obeythetestinggoat-book-tester:latest . + +.PHONY: docker-push +docker-push: + docker push hjwp/obeythetestinggoat-book-tester:latest + +.PHONY: docker-venv +docker-venv: + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + --mount type=volume,source=venvmount,target=/app/.venv \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "uv venv .venv && uv pip install --upgrade ." + +.PHONY: docker-test_% +docker-test_%: docker-venv %.html $(TMPDIR) + docker run --rm \ + --mount type=volume,source=rootmount,target=/root \ + --mount type=bind,source=./,target=/app \ + --mount type=volume,source=venvmount,target=/app/.venv \ + -t hjwp/obeythetestinggoat-book-tester:latest \ + bash -c "make $(subst docker-,,$@)" + From 1cf9cc25a60b434cfcc3c599fc4dd214230b34a1 Mon Sep 17 00:00:00 2001 From: Harry Date: Wed, 7 Aug 2024 13:14:40 +0100 Subject: [PATCH 9/9] try doing venv inside /app --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e8aaf745c..be543d924 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,9 +24,9 @@ RUN git config --global user.email "elspeth@example.com" && \ WORKDIR /app COPY pyproject.toml pyproject.toml -# install python deps systemwide with uv -ENV PIP_CACHE_DIR=/var/cache/pip - -RUN uv pip install --system . +# install python deps into a venv at /app/.venv, +# as a form of cache. even when we mount over it with --volume, +# the content is copied in. +RUN uv venv .venv && uv pip install . CMD bash