Initial Py11 base

.github/workflows/build.yaml

@@ -10,7 +10,44 @@ env:
   BASE_TAG: 4.4.0.stable
 
 jobs:
+  build-base:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    defaults:
+      run:
+        working-directory: al-service-with-py11
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Authorize to GitHub Packages
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: Check if the current version has already been pushed
+        id: check-if-pushed
+        run: |
+          export GHCR_TOKEN=$(echo ${{ secrets.GITHUB_TOKEN }} | base64)
+          export TAG=$BASE_TAG$(cat BASE_VERSION)
+          echo manifest_base=$(curl -s -H "Authorization: Bearer ${GHCR_TOKEN}" https://ghcr.io/v2/kam193/assemblyline-base-py11/manifests/$TAG | grep "manifest unknown") > $GITHUB_OUTPUT
+          export TAG=$BASE_TAG$(cat VERSION)
+          echo manifest_service=$(curl -s -H "Authorization: Bearer ${GHCR_TOKEN}" https://ghcr.io/v2/kam193/assemblyline-service-base-py11/manifests/$TAG | grep "manifest unknown") > $GITHUB_OUTPUT
+      - name: Build base and service image
+        if: steps.check-if-pushed.outputs.manifest_base || steps.check-if-pushed.outputs.manifest_service
+        run: |
+          make build-base
+          make build-service
+      - name: Push base image
+        if: steps.check-if-pushed.outputs.manifest_base
+        run: |
+          make push-base
+      - name: Push service image
+        if: steps.check-if-pushed.outputs.manifest_service
+        run: |
+          make push-service
   discover-services:
+    needs: build-base
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -20,7 +57,7 @@ jobs:
       - name: Discover directories with Dockerfiles
         id: services
         run: |
-          echo "services=$(find . -type f -name Dockerfile | xargs -n 1 dirname | uniq | cut -d '/' -f 2 | grep -v 'TEMPLATE' | jq -R -s -c 'split("\n")[:-1]')" >> "$GITHUB_OUTPUT"
+          echo "services=$(find . -type f -name Dockerfile | xargs -n 1 dirname | uniq | cut -d '/' -f 2 | grep -v 'TEMPLATE' | grep -v 'al-service-with-py11' | jq -R -s -c 'split("\n")[:-1]')" >> "$GITHUB_OUTPUT"
       - name: Print services with Dockerfiles
         run: |
           echo "Services with Dockerfiles: ${{ steps.services.outputs.services }}"

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "al-service-with-py11/assemblyline-base"]
+	path = al-service-with-py11/assemblyline-base
+	url = https://github.com/CybercentreCanada/assemblyline-base.git
+[submodule "al-service-with-py11/assemblyline-v4-service"]
+	path = al-service-with-py11/assemblyline-v4-service
+	url = https://github.com/CybercentreCanada/assemblyline-v4-service.git

al-service-with-py11/.dockerignore

@@ -0,0 +1,3 @@
+.env
+.randomnotes/
+.git/

al-service-with-py11/BASE_VERSION

@@ -0,0 +1 @@
+4

al-service-with-py11/Dockerfile.base

@@ -0,0 +1,63 @@
+ARG REGISTRY
+FROM ${REGISTRY}python:3.11-slim-bookworm AS base
+
+# Upgrade packages
+RUN apt-get update && apt-get -yy upgrade && rm -rf /var/lib/apt/lists/*
+
+# Get required apt packages
+RUN apt-get update && apt-get install -yy libffi8 libfuzzy2 libmagic1 libssl-dev && rm -rf /var/lib/apt/lists/*
+
+# Make sure root account is locked so 'su' commands fail all the time
+RUN passwd -l root
+
+FROM base AS builder
+ARG version
+ARG version_tag=${version}
+
+# Get required apt packages
+RUN apt-get update \
+  && apt-get install -yy build-essential libffi-dev libfuzzy-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+# Install assemblyline base (setup.py is just a file we know exists so the command
+# won't fail if dist isn't there. The dist* copies in any dist directory only if it exists.)
+COPY setup.py dist* dist/
+RUN pip install --no-cache-dir --no-warn-script-location -f dist/ --user assemblyline>=$version && rm -rf ~/.cache/pip
+RUN chmod 750 /root/.local/lib/python3.11/site-packages
+
+FROM base
+
+# Add assemblyline user
+RUN useradd -b /var/lib -U -m assemblyline
+
+# Create assemblyline config directory
+RUN mkdir -p /etc/assemblyline
+RUN chmod 750 /etc/assemblyline
+RUN chown root:assemblyline /etc/assemblyline
+
+# Create assemblyline cache directory
+RUN mkdir -p /var/cache/assemblyline
+RUN chmod 770 /var/cache/assemblyline
+RUN chown assemblyline:assemblyline /var/cache/assemblyline
+
+# Create assemblyline home directory
+RUN mkdir -p /var/lib/assemblyline
+RUN chmod 750 /var/lib/assemblyline
+RUN chown assemblyline:assemblyline /var/lib/assemblyline
+
+# Create assemblyline log directory
+RUN mkdir -p /var/log/assemblyline
+RUN chmod 770 /var/log/assemblyline
+RUN chown assemblyline:assemblyline /var/log/assemblyline
+
+# Install assemblyline base
+COPY --chown=assemblyline:assemblyline --from=builder /root/.local /var/lib/assemblyline/.local
+ENV PATH=/var/lib/assemblyline/.local/bin:$PATH
+ENV PYTHONPATH=/var/lib/assemblyline/.local/lib/python3.11/site-packages
+ENV ASSEMBLYLINE_VERSION=${version}
+ENV ASSEMBLYLINE_IMAGE_TAG=${version_tag}
+
+# Switch to assemblyline user
+USER assemblyline
+WORKDIR /var/lib/assemblyline
+CMD /bin/bash

al-service-with-py11/Dockerfile.service

@@ -0,0 +1,64 @@
+ARG REGISTRY
+ARG branch=latest
+ARG base=cccs/assemblyline
+FROM ${REGISTRY}$base:$branch as builder
+ARG version
+ARG branch
+
+USER root
+RUN apt-get update \
+  && apt-get install -yy build-essential libfuzzy-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+# install pip packages, as assemblyline so they go into our .local with the ones already there
+USER assemblyline
+RUN touch /tmp/before-pip
+COPY setup.py dist* dist/
+RUN pip install --no-cache-dir -f dist --user \
+  assemblyline-core>=version \
+  assemblyline-service-client>=version \
+  assemblyline-v4-service>=version \
+  assemblyline-client \
+  gunicorn[gevent] flask gitpython git-remote-codecommit psutil \
+  && rm -rf ~/.cache/pip
+
+# If this is a latest rather than stable build, try to push for unstable packages of assemblyline dependencies
+RUN if [ "${branch}" = "latest" ]; then pip install --user --upgrade --pre assemblyline-client; fi
+
+# Remove files that existed before the pip install so that our copy command below doesn't take a snapshot of
+# files that already exist in the base image
+RUN find /var/lib/assemblyline/.local -type f ! -newer /tmp/before-pip -delete
+
+# Switch back to root and change the ownership of the files to be copied due to bitbucket pipeline uid nonsense
+USER root
+RUN chown root:root -R /var/lib/assemblyline/.local
+
+# Restart a new image, this time the output one
+ARG REGISTRY
+ARG base=cccs/assemblyline
+FROM ${REGISTRY}$base:$branch
+
+# Get the updated local dir from builder
+COPY --chown=assemblyline:assemblyline --from=builder /var/lib/assemblyline/.local /var/lib/assemblyline/.local
+
+# Setup environment varibles
+ENV PYTHONPATH /opt/al_service
+ENV SERVICE_API_HOST http://al_service_server:5003
+ENV SERVICE_API_KEY ThisIsARandomAuthKey...ChangeMe!
+ENV CONTAINER_MODE true
+
+USER root
+# Install git binary for gitpython pip package
+RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+
+# Create directory for service
+RUN mkdir -p /opt/al_service
+RUN touch /opt/al_service/__init__.py
+COPY docker/process_handler.py /etc/
+
+# Create directory for service updates
+RUN mkdir /updates
+RUN chown -R assemblyline:assemblyline /updates
+
+USER assemblyline
+CMD ["python", "/etc/process_handler.py"]

al-service-with-py11/LICENSE

@@ -0,0 +1,4 @@
+Dockerfiles are based on the original files from AssemblyLine project
+and are licensed under MIT license.
+
+See: https://github.com/CybercentreCanada/assemblyline

al-service-with-py11/Makefile

@@ -0,0 +1,41 @@
+include ../common.mk
+
+SERVICE_NAME=assemblyline-service-base-py11
+
+AL_VERSION=4.4.0
+BASE_NAME=assemblyline-base-py11
+
+BASE_TAG=$(shell cat BASE_VERSION)
+CACHE=
+
+manifest:
+	true
+
+build-base:
+	docker build -t kam193/${BASE_NAME}:latest --build-arg version=${AL_VERSION} --build-arg REGISTRY=${REGISTRY}/ -f Dockerfile.base ${CACHE} ./assemblyline-base
+	docker tag kam193/${BASE_NAME}:latest ${REGISTRY}/kam193/${BASE_NAME}:latest
+
+bump_version-base:
+	NEW_TAG=$$((${BASE_TAG}+1)) && echo $$NEW_TAG > BASE_VERSION
+
+push-base: build-base
+	docker tag kam193/${BASE_NAME}:latest ${PUSH_REGISTRY}/kam193/${BASE_NAME}:latest
+	docker tag kam193/${BASE_NAME}:latest ${PUSH_REGISTRY}/kam193/${BASE_NAME}:${BASE_TAG}$$(cat BASE_VERSION)
+	docker push ${PUSH_REGISTRY}/kam193/${BASE_NAME}:latest
+	docker push ${PUSH_REGISTRY}/kam193/${BASE_NAME}:${BASE_TAG}$$(cat BASE_VERSION)
+
+build-service:
+	docker build -t kam193/${SERVICE_NAME}:latest --build-arg REGISTRY=${REGISTRY}/ --build-arg version=${AL_VERSION} --build-arg base=kam193/${BASE_NAME} -f Dockerfile.service ${CACHE} ./assemblyline-v4-service
+	docker tag kam193/${SERVICE_NAME}:latest ${REGISTRY}/kam193/${SERVICE_NAME}:latest
+
+bump_version-service:
+	NEW_TAG=$$((${TAG}+1)) && echo $$NEW_TAG > VERSION
+
+push-service: build-service
+	docker tag kam193/${SERVICE_NAME}:latest ${PUSH_REGISTRY}/kam193/${SERVICE_NAME}:latest
+	docker tag kam193/${SERVICE_NAME}:latest ${PUSH_REGISTRY}/kam193/${SERVICE_NAME}:${BASE_TAG}$$(cat VERSION)
+	docker push ${PUSH_REGISTRY}/kam193/${SERVICE_NAME}:${BASE_TAG}$$(cat VERSION)
+	docker push ${PUSH_REGISTRY}/kam193/${SERVICE_NAME}:latest
+
+refresh-service: CACHE="--no-cache"
+refresh-service: build-service

al-service-with-py11/VERSION

@@ -0,0 +1 @@
+7

pcap-extractor/Makefile

@@ -2,4 +2,4 @@ include ../common.mk
 
 AL_SERVICE_NAME=PCAP-Extractor
 # SERVICE_NAME=assemblyline-service-template
-BASE_IMAGE=kam193/assemblyline-service-base-py11:latest
+BASE_IMAGE=${REGISTRY}/kam193/assemblyline-service-base-py11:latest