[PAN-1863] test debian package (#30)

* fix: PAN-1863 test debian package * fix: PAN-1971 include python dependencies on build * fix: PAN-1971 configurator * fix: PAN-1971 add conda build * fix: PAN-1971 mod-wsgi * fix: PAN-1971 rollback common and comments
pantos-io · Jun 24, 2024 · 895ad9f · 895ad9f
1 parent 2cf9f50
commit 895ad9f
Show file tree

Hide file tree

Showing 51 changed files with 1,107 additions and 352 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -7,15 +7,15 @@ on:
         required: false
         type: string
         default: "1.0.0"
-      revision:
-        description: 'Revision to build'
-        required: false
-        type: string
-        default: ${{ github.run_attempt }}
       environment:
         description: 'Environment where the secrets are stored'
         required: false
         type: string
+      architecture:
+        description: 'Architecture to build'
+        required: false
+        type: string
+        default: "amd64"
     secrets:
       GPG_PRIVATE_KEY:
         description: 'GPG private key'
@@ -27,8 +27,13 @@ on:
 jobs:
   build-deb:
     name: Build and attach .deb and .whl packages
-    runs-on: ubuntu-latest
+    # TODO: Change ubuntu-20.04 for the ARM public runner
+    runs-on: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }}
     environment: ${{ inputs.environment }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"]
     steps:
       - uses: step-security/harden-runner@v2
         with:
@@ -38,6 +43,9 @@ jobs:
 
       - name: Set up Poetry
         uses: pantos-io/ci-workflows/.github/actions/install-poetry@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+          runner-os: ${{ inputs.architecture == 'amd64' && 'ubuntu-latest' || 'ubuntu-20.04' }}
 
       - name: Check secrets
         id: is-signed-build
@@ -55,19 +63,46 @@ jobs:
           gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
           passphrase: ${{ secrets.GPG_PASSPHRASE }}
 
+      - name: Install conda dependencies
+        run: |
+          ARCH=$(uname -m)
+          if [ "$ARCH" = "x86_64" ]; then
+              MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh";
+          elif [ "$ARCH" = "aarch64" ]; then
+              MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh";
+          else
+              echo "Unsupported architecture: $ARCH";
+              exit 1;
+          fi
+          wget "$MINICONDA_URL" -O miniconda.sh
+          bash miniconda.sh -b
+          rm -f miniconda.sh
+        shell: sh
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install build-essential debhelper devscripts equivs dh-virtualenv python3-venv dh-sysuser dh-exec -y
+          sudo make debian-build-deps
+
       - name: Install signing dependencies
         if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
         run: |
           sudo apt-get update
           sudo apt-get install -y debsigs
 
+      - name: Ensure version
+        if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
+        run: |
+          make check-version VERSION=${{ inputs.version }}
+
       - name: Build package
         run: |
-          make debian
+          make debian debian-full
           make wheel
         env:
           PANTOS_SERVICE_NODE_VERSION: ${{ inputs.version }}
-          PANTOS_SERVICE_NODE_REVISION: ${{ inputs.revision }}
+        shell: sh
 
       - name: Sign package
         if: steps.is-signed-build.outputs.HAS_SECRETS == 'true'
@@ -77,5 +112,5 @@ jobs:
       - name: Upload artifact
         uses: actions/upload-artifact@v4
         with:
-          name: build
+          name: build-${{ inputs.architecture }}
           path: dist/*
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -60,8 +60,8 @@ jobs:
           python3 -m venv .venv
           source .venv/bin/activate
           pip3 install pycryptodome==3.18.0
-          chmod +x scripts/generate-signer-key.py
-          cat password.key | ./scripts/generate-signer-key.py
+          chmod +x ./.github/workflows/scripts/generate-signer-key.py
+          cat password.key | ./.github/workflows/scripts/generate-signer-key.py
           mv signer-key*.pem signer_key.pem
 
       # - name: Test image
@@ -74,5 +74,53 @@ jobs:
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
   build:
-    needs: [shared]
     uses: ./.github/workflows/build.yml
+    with:
+      architecture: amd64
+
+  install:
+    needs: [build]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os:
+          - ubuntu-20.04
+          - ubuntu-22.04
+          - ubuntu-24.04
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: build-amd64
+          path: build
+
+      - name: Preconfigure package
+        run: |
+          echo 'pantos-service-node-full pantos/service-node/create_signer_key boolean true' | sudo debconf-set-selections;
+          echo 'pantos-service-node-full pantos/service-node/signer_key_password string 1234' | sudo debconf-set-selections;
+          echo 'pantos-service-node-full pantos/common/configure_postgresql boolean true' | sudo debconf-set-selections;
+          echo 'pantos-service-node-full pantos/common/configure_rabbitmq boolean true' | sudo debconf-set-selections;
+          echo 'pantos-service-node-full pantos/common/reset_users boolean true' | sudo debconf-set-selections;
+          echo 'pantos-service-node-full pantos/common/start_services boolean false' | sudo debconf-set-selections;
+
+      - name: Install Debian package
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ./build/*.deb
+
+      # Check service is running
+      - name: Check service is running
+        run: |
+          sudo systemctl status pantos-service-node-server || true
+          sudo systemctl status pantos-service-node-celery || true
+          sudo journalctl -u pantos-service-node-server || true
+          sudo journalctl -u pantos-service-node-celery || true
+
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: install-logs-${{ matrix.os }}
+          path: /var/log/pantos
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -89,7 +89,8 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          name: build
+          merge-multiple: true
+          pattern: build-
           path: dist
 
       - name: Rename built artifacts
@@ -145,8 +146,8 @@ jobs:
     with:
       # We need to use a semver that doesn't start with a v as debian will remove it anyways
       version: ${{ needs.define-environment.outputs.deployment_version }}
-      revision: "${{ github.event.release.prerelease && 'rc' || '' }}${{ github.run_attempt }}"
       environment: debian-release
+      architecture: amd64
 
   add-assets:
     name: Add Assets to the ${{ github.event.release.tag_name }} Release
@@ -162,7 +163,8 @@ jobs:
           egress-policy: audit
       - uses: actions/download-artifact@v4
         with:
-          name: build
+          merge-multiple: true
+          pattern: build-
           path: dist
 
       # We need to upload some artifacts, any, so that the download action works
@@ -264,7 +266,8 @@ jobs:
           egress-policy: audit
       - uses: actions/download-artifact@v4
         with:
-          name: build
+          merge-multiple: true
+          pattern: build-
           path: dist
 
       - name: List directory

diff --git a/scripts/generate-signer-key.py → .../workflows/scripts/generate-signer-key.py b/scripts/generate-signer-key.py → .../workflows/scripts/generate-signer-key.py
diff --git a/.gitignore b/.gitignore
@@ -13,3 +13,9 @@ find.sh
 local/
 postgresql_data
 rabbitmq_data
+debian/.debhelper
+debian/pantos-service-node
+debian/tmp
+debian/*debhelper*
+debian/files
+debian/*substvars
diff --git a/Dockerfile b/Dockerfile
@@ -1,18 +1,41 @@
 FROM python:3.12-bookworm AS dev
 
+RUN apt-get update && \
+    apt-get install build-essential debhelper devscripts \
+    equivs dh-virtualenv python3-venv dh-sysuser dh-exec \
+    -y --no-install-recommends
+
+ENV PATH="/root/miniconda3/bin:${PATH}"
+RUN ARCH=$(uname -m) && \
+    if [ "$ARCH" = "x86_64" ]; then \
+        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh"; \
+    elif [ "$ARCH" = "aarch64" ]; then \
+        MINICONDA_URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-aarch64.sh"; \
+    else \
+        echo "Unsupported architecture: $ARCH"; \
+        exit 1; \
+    fi && \
+    wget "$MINICONDA_URL" -O miniconda.sh && \
+    mkdir /root/.conda && \
+    bash miniconda.sh -b && \
+    rm -f miniconda.sh
+
 RUN python3 -m pip install poetry
 
 WORKDIR /app
 
 COPY . /app
 
+RUN make debian-build-deps
+
 RUN make debian
 
 FROM bitnami/minideb:bookworm AS prod
 
 RUN apt-get update
 
-COPY --from=dev /app/dist/*.deb .
+# Do not copy the configurator package
+COPY --from=dev /app/dist/pantos-service-node_*.deb .
 
 RUN if [ -f ./*-signed.deb ]; then \
         apt-get install -y --no-install-recommends ./*-signed.deb; \
@@ -27,9 +50,9 @@ FROM prod AS servicenode
 
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "python", "-c", 'import requests; response = requests.get("http://localhost:8080/health/live"); response.raise_for_status();' ]
 
-ENTRYPOINT bash -c 'source /opt/pantos/service-node/virtual-environment/bin/activate && \
+ENTRYPOINT bash -c 'source /opt/pantos/pantos-service-node/virtual-environment/bin/activate && \
     exec mod_wsgi-express start-server \
-    /opt/pantos/service-node/wsgi.py \
+    /opt/pantos/pantos-service-node/wsgi.py \
     --user pantos \
     --group pantos \
     --port 8080 \
@@ -40,7 +63,7 @@ FROM prod AS servicenode-celery-worker
 
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "bash", "-c", 'celery inspect ping -A pantos.servicenode -d celery@\$HOSTNAME' ]
 
-ENTRYPOINT bash -c 'source /opt/pantos/service-node/virtual-environment/bin/activate && \
+ENTRYPOINT bash -c 'source /opt/pantos/pantos-service-node/virtual-environment/bin/activate && \
     celery \
     -A pantos.servicenode \
     worker \

diff --git a/Makefile b/Makefile
@@ -1,9 +1,22 @@
 PANTOS_SERVICE_NODE_VERSION := $(shell poetry version -s)
-PANTOS_SERVICE_NODE_REVISION ?= 1
 PANTOS_SERVICE_NODE_SSH_HOST ?= bdev-service-node
-PYTHON_FILES_WITHOUT_TESTS := pantos/servicenode linux/start-web-server
+PYTHON_FILES_WITHOUT_TESTS := pantos/servicenode linux/scripts/start-web.py
 PYTHON_FILES := $(PYTHON_FILES_WITHOUT_TESTS) tests
 
+.PHONY: check-version
+check-version:
+	@if [ -z "$(VERSION)" ]; then \
+		echo "Error: VERSION is not set"; \
+		exit 1; \
+	fi
+	@VERSION_FROM_POETRY=$$(poetry version -s) ; \
+	if test "$$VERSION_FROM_POETRY" != "$(VERSION)"; then \
+		echo "Version mismatch: expected $(VERSION), got $$VERSION_FROM_POETRY" ; \
+		exit 1 ; \
+	else \
+		echo "Version check passed" ; \
+	fi
+
 .PHONY: dist
 dist: tar wheel debian
 
@@ -83,8 +96,19 @@ dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION).tar.gz: pantos/ service-
 	rm pantos/pantos-service-node.sh
 	rm pantos/pantos-service-node-worker.sh
 
+check-poetry-plugin:
+	@if poetry self show plugins | grep -q poetry-plugin-freeze; then \
+		echo "poetry-plugin-freeze is already added."; \
+	else \
+		echo "poetry-plugin-freeze is not added. Adding now..."; \
+		poetry self add poetry-plugin-freeze; \
+	fi
+
+freeze-wheel: check-poetry-plugin
+	poetry freeze-wheel
+
 .PHONY: wheel
-wheel: dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION)-py3-none-any.whl
+wheel: dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION)-py3-none-any.whl freeze-wheel
 
 dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION)-py3-none-any.whl: pantos/ service-node-config.yml service-node-config.env bids.yml alembic.ini
 	cp service-node-config.yml pantos/service-node-config.yml
@@ -97,33 +121,29 @@ dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION)-py3-none-any.whl: pantos
 	rm pantos/bids.yml
 	rm pantos/alembic.ini
 
+.PHONY: debian-build-deps
+debian-build-deps:
+	mk-build-deps --install --tool "apt-get --no-install-recommends -y" debian/control --remove
+
+debian-full:
+	mkdir -p dist
+	sed 's/VERSION_PLACEHOLDER/$(PANTOS_SERVICE_NODE_VERSION)/' configurator/DEBIAN/control.template > configurator/DEBIAN/control
+	dpkg-deb --build configurator dist/pantos-service-node-full_$(PANTOS_SERVICE_NODE_VERSION)_all.deb
+	rm configurator/DEBIAN/control
+
 .PHONY: debian
-debian: dist/pantos-service-node-$(PANTOS_SERVICE_NODE_VERSION)-$(PANTOS_SERVICE_NODE_REVISION)_all.deb
-
-dist/pantos-service-node-$(PANTOS_SERVICE_NODE_VERSION)-$(PANTOS_SERVICE_NODE_REVISION)_all.deb: linux/ dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION)-py3-none-any.whl
-	$(eval debian_package := pantos-service-node-$(PANTOS_SERVICE_NODE_VERSION)-$(PANTOS_SERVICE_NODE_REVISION)_all)
-	$(eval build_directory := build/debian/$(debian_package))
-	mkdir -p $(build_directory)/opt/pantos/service-node
-	cp dist/pantos_service_node-$(PANTOS_SERVICE_NODE_VERSION)-py3-none-any.whl $(build_directory)/opt/pantos/service-node/
-	cp linux/start-web-server $(build_directory)/opt/pantos/service-node/
-	mkdir -p $(build_directory)/etc/systemd/system
-	cp linux/pantos-service-node-server.service $(build_directory)/etc/systemd/system/
-	cp linux/pantos-service-node-celery.service $(build_directory)/etc/systemd/system/
-	mkdir -p $(build_directory)/DEBIAN
-	cat linux/debian/control | sed -e 's/VERSION/$(PANTOS_SERVICE_NODE_VERSION)/g' > $(build_directory)/DEBIAN/control
-	cat linux/debian/postinst | sed -e 's/VERSION/$(PANTOS_SERVICE_NODE_VERSION)/g' > $(build_directory)/DEBIAN/postinst
-	cp linux/debian/prerm $(build_directory)/DEBIAN/prerm
-	cp linux/debian/postrm $(build_directory)/DEBIAN/postrm
-	chmod 755 $(build_directory)/DEBIAN/postinst
-	chmod 755 $(build_directory)/DEBIAN/prerm
-	chmod 755 $(build_directory)/DEBIAN/postrm
-	cd build/debian/; \
-		dpkg-deb --build --root-owner-group -Zgzip $(debian_package)
-	mv build/debian/$(debian_package).deb dist/
+debian:
+	$(eval debian_package := pantos-service-node_$(PANTOS_SERVICE_NODE_VERSION)_*.deb)
+	dpkg-buildpackage -uc -us -g
+	mkdir -p dist
+	mv ../$(debian_package) dist/
+
+debian-all: debian debian-full
+
 
 .PHONY: remote-install
-remote-install: dist/pantos-service-node-$(PANTOS_SERVICE_NODE_VERSION)-$(PANTOS_SERVICE_NODE_REVISION)_all.deb
-	$(eval deb_file := pantos-service-node-$(PANTOS_SERVICE_NODE_VERSION)-$(PANTOS_SERVICE_NODE_REVISION)_all.deb)
+remote-install: debian-all
+	$(eval deb_file := pantos-service-node*_$(PANTOS_SERVICE_NODE_VERSION)_*.deb)
 	scp dist/$(deb_file) $(PANTOS_SERVICE_NODE_SSH_HOST):
 	ssh -t $(PANTOS_SERVICE_NODE_SSH_HOST) "\
 		sudo systemctl stop pantos-service-node-celery;\

diff --git a/configurator/DEBIAN/.gitignore b/configurator/DEBIAN/.gitignore
@@ -0,0 +1,7 @@
+.debhelper
+pantos-service-node
+tmp
+*.debhelper
+*.substvars
+debhelper*
+files
diff --git a/configurator/DEBIAN/compat b/configurator/DEBIAN/compat
@@ -0,0 +1 @@
+10