diff --git a/.github/workflows/baremetal-deploy.yml b/.github/workflows/baremetal-deploy.yml index 1dbb50d..9b22735 100644 --- a/.github/workflows/baremetal-deploy.yml +++ b/.github/workflows/baremetal-deploy.yml @@ -1,11 +1,11 @@ name: GHAE - deploy runners (bare metal) on: - workflow_dispatch: # deploy on demand + workflow_dispatch: # deploy on demand jobs: deploy: - runs-on: ubuntu-latest # use the GitHub hosted runners to deploy the self-hosted runners in GHEC + runs-on: ubuntu-latest # use the GitHub hosted runners to deploy the self-hosted runners in GHEC # If using GHES or GHAE, use another deployment, such as having CentOS redeploy Ubuntu and vice versa environment: bare-metal @@ -14,13 +14,15 @@ jobs: uses: actions/checkout@v3 - name: Write out the kubeconfig info - run: | + run: | echo ${{ secrets.DEPLOY_ACCOUNT }} | base64 -d > /tmp/config - name: Update deployment run: | kubectl apply -f deployments/ghae/podman.yml --insecure-skip-tls-verify kubectl apply -f deployments/ghae/ubuntu-focal.yml --insecure-skip-tls-verify + kubectl apply -f deployments/ghae/rootless-ubuntu-focal.yml --insecure-skip-tls-verify + kubectl apply -f deployments/ghae/ubuntu-jammy.yml --insecure-skip-tls-verify env: KUBECONFIG: /tmp/config diff --git a/.github/workflows/baremetal-remove.yml b/.github/workflows/baremetal-remove.yml index 6c51cb0..99c8b98 100644 --- a/.github/workflows/baremetal-remove.yml +++ b/.github/workflows/baremetal-remove.yml @@ -1,11 +1,11 @@ name: GHAE - remove runners (bare metal) on: - workflow_dispatch: # deploy on demand + workflow_dispatch: # deploy on demand jobs: deploy: - runs-on: ubuntu-latest # use the GitHub hosted runners to deploy the self-hosted runners in GHEC + runs-on: ubuntu-latest # use the GitHub hosted runners to deploy the self-hosted runners in GHEC # If using GHES or GHAE, use another deployment, such as having CentOS redeploy Ubuntu and vice versa environment: bare-metal @@ -14,13 +14,15 @@ jobs: uses: actions/checkout@v3 - name: Write out the kubeconfig info - run: | + run: | echo ${{ secrets.DEPLOY_ACCOUNT }} | base64 -d > /tmp/config - name: Update deployment run: | kubectl delete -f deployments/ghae/podman.yml --insecure-skip-tls-verify kubectl delete -f deployments/ghae/ubuntu-focal.yml --insecure-skip-tls-verify + kubectl delete -f deployments/ghae/rootless-ubuntu-focal.yml --insecure-skip-tls-verify + kubectl delete -f deployments/ghae/ubuntu-jammy.yml --insecure-skip-tls-verify env: KUBECONFIG: /tmp/config diff --git a/.github/workflows/build-release.yml b/.github/workflows/build-release.yml index f9ec038..d4c70d0 100644 --- a/.github/workflows/build-release.yml +++ b/.github/workflows/build-release.yml @@ -1,20 +1,20 @@ name: Build/publish/deploy all runners on: - workflow_dispatch: # build on demand + workflow_dispatch: # build on demand release: - types: [published] # build on release + types: [published] # build on release jobs: build-ubuntu: - runs-on: ubuntu-latest # use the GitHub hosted runners + runs-on: ubuntu-latest # use the GitHub hosted runners permissions: - contents: write # for uploading the SBOM to the release - packages: write # for uploading the finished container - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + contents: write # for uploading the SBOM to the release + packages: write # for uploading the finished container + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results strategy: matrix: - runner-image: [ ubuntu-focal, rootless-ubuntu-focal ] + runner-image: [ubuntu-focal, rootless-ubuntu-focal, ubuntu-jammy] steps: - name: Checkout @@ -26,7 +26,7 @@ jobs: - name: Set outputs id: vars run: echo ::set-output name=sha_short::${GITHUB_SHA::7} - + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -40,7 +40,7 @@ jobs: - name: Build and push the image uses: docker/build-push-action@v3 with: - file: 'images/${{ matrix.runner-image }}.Dockerfile' + file: "images/${{ matrix.runner-image }}.Dockerfile" push: true tags: | ghcr.io/some-natalie/kubernoodles/${{ matrix.runner-image }}:latest @@ -60,17 +60,17 @@ jobs: with: sarif_file: ${{ steps.scan.outputs.sarif }} - - name: Generate SBOM for the rootless Ubuntu Focal (20.04 LTS) runner + - name: Generate SBOM for the Ubuntu-based runners uses: anchore/sbom-action@v0 with: image: ghcr.io/some-natalie/kubernoodles/${{ matrix.runner-image }}:latest build-podman: - runs-on: ubuntu-latest # use the GitHub hosted runners + runs-on: ubuntu-latest # use the GitHub hosted runners permissions: - contents: write # for uploading the SBOM to the release - packages: write # for uploading the finished container - security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + contents: write # for uploading the SBOM to the release + packages: write # for uploading the finished container + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results steps: - name: Checkout @@ -82,7 +82,7 @@ jobs: - name: Set outputs id: vars run: echo ::set-output name=sha_short::${GITHUB_SHA::7} - + - name: Build the image id: build-image uses: redhat-actions/buildah-build@v2 @@ -108,7 +108,7 @@ jobs: uses: anchore/sbom-action@v0 with: image: ghcr.io/some-natalie/kubernoodles/podman:latest - + - name: Push image uses: redhat-actions/push-to-registry@v2 with: @@ -119,20 +119,20 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} deploy: - runs-on: ubuntu-latest # use the GitHub hosted runners to deploy the self-hosted runners in GHEC + runs-on: ubuntu-latest # use the GitHub hosted runners to deploy the self-hosted runners in GHEC # If using GHES or GHAE, use another deployment, such as having CentOS redeploy Ubuntu and vice versa environment: production - needs: [ build-podman, build-ubuntu ] + needs: [build-podman, build-ubuntu] strategy: matrix: - runner-image: [ podman, ubuntu-focal, rootless-ubuntu-focal ] + runner-image: [podman, ubuntu-focal, rootless-ubuntu-focal] steps: - name: Checkout uses: actions/checkout@v3 - name: Write out the kubeconfig info - run: | + run: | echo ${{ secrets.DEPLOY_ACCOUNT }} | base64 -d > /tmp/config - name: Update deployment diff --git a/.github/workflows/test-ubuntu-jammy.yml b/.github/workflows/test-ubuntu-jammy.yml new file mode 100644 index 0000000..c31cfa0 --- /dev/null +++ b/.github/workflows/test-ubuntu-jammy.yml @@ -0,0 +1,114 @@ +name: Test Ubuntu Jammy runner + +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - "images/ubuntu-jammy.Dockerfile" + - "images/**.sh" + - "images/docker/*" + - "images/software/*" + - "images/supervisor/*" + +jobs: + build: + name: Build test image + runs-on: [self-hosted, jammy] + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to GitHub Packages + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + uses: docker/build-push-action@v3 + with: + file: "images/ubuntu-jammy.Dockerfile" + push: true + tags: ghcr.io/some-natalie/kubernoodles/ubuntu-jammy:latest + + deploy: + name: Deploy test image to `test-runners` namespace + runs-on: [self-hosted, jammy] + needs: [build] + environment: test + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Write out the kubeconfig info + run: | + echo ${{ secrets.DEPLOY_ACCOUNT }} | base64 -d > /tmp/config + + - name: Deploy + run: | + kubectl apply -f deployments/test-ubuntu-jammy.yml + env: + KUBECONFIG: /tmp/config + + - name: Remove kubeconfig info + run: rm -f /tmp/config + + - name: Wait 5 minutes to let the new pod come up + run: sleep 300 + + test: + name: Run tests! + runs-on: [self-hosted, test-ubuntu-jammy] + needs: [deploy] + + steps: + - name: Sudo test + run: sudo echo "sudo is working" + + - name: Docker test + run: | + docker run hello-world + docker network inspect bridge + docker info + + - name: Docker compose test + run: | + docker-compose --version + + - name: Print environmental variables + run: printenv + + remove-deploy: + name: Delete test image deployment + runs-on: [self-hosted, jammy] + needs: [test] + environment: test + if: always() + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Write out the kubeconfig info + run: | + echo ${{ secrets.DEPLOY_ACCOUNT }} | base64 -d > /tmp/config + + - name: Deploy + run: | + kubectl delete -f deployments/test-ubuntu-jammy.yml + env: + KUBECONFIG: /tmp/config + + - name: Remove kubeconfig info + run: rm -f /tmp/config diff --git a/.github/workflows/weekly-cleanup.yml b/.github/workflows/weekly-cleanup.yml index eda60ff..82d49b0 100644 --- a/.github/workflows/weekly-cleanup.yml +++ b/.github/workflows/weekly-cleanup.yml @@ -3,7 +3,7 @@ name: Weekly repo cleanup 🔥 on: workflow_dispatch: schedule: - - cron: '30 22 * * 1' # Weekly at 22:30 UTC on Mondays + - cron: "30 22 * * 1" # Weekly at 22:30 UTC on Mondays jobs: clean-ghcr: @@ -16,6 +16,7 @@ jobs: - ubuntu-focal - podman - rootless-ubuntu-focal + - ubuntu-jammy steps: - name: Delete untagged containers uses: snok/container-retention-policy@v1 @@ -34,18 +35,18 @@ jobs: - name: Close stale issues and pull requests uses: actions/stale@v5 with: - stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 2 weeks.' - close-issue-message: 'This issue was closed because it has been stalled for 2 weeks with no activity.' + stale-issue-message: "This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 2 weeks." + close-issue-message: "This issue was closed because it has been stalled for 2 weeks with no activity." days-before-issue-stale: 30 days-before-issue-close: 14 - stale-issue-label: 'stale' - exempt-issue-labels: 'epic' - stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 2 weeks.' - close-pr-message: 'This PR was closed because it has been stalled for 2 weeks with no activity.' + stale-issue-label: "stale" + exempt-issue-labels: "epic" + stale-pr-message: "This PR is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 2 weeks." + close-pr-message: "This PR was closed because it has been stalled for 2 weeks with no activity." days-before-pr-stale: 30 days-before-pr-close: 14 - stale-pr-label: 'stale' - exempt-pr-labels: 'dependencies' + stale-pr-label: "stale" + exempt-pr-labels: "dependencies" clean-offline-runners: name: Delete offline self-hosted runners diff --git a/deployments/ghae/ubuntu-jammy.yml b/deployments/ghae/ubuntu-jammy.yml new file mode 100644 index 0000000..5417cec --- /dev/null +++ b/deployments/ghae/ubuntu-jammy.yml @@ -0,0 +1,55 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: ubuntu-jammy + namespace: runners +spec: + replicas: 1 + template: + spec: + organization: universal-exports-ltd + # env: + # - name: DISABLE_RUNNER_UPDATE # Disables automatic runner updates + # value: "true" + ephemeral: true + image: ghcr.io/some-natalie/kubernoodles/ubuntu-jammy:latest # change this to the version you really want! + imagePullPolicy: Always + imagePullSecrets: + - name: ghcr + dockerdWithinRunnerContainer: true + dockerMTU: 1450 + resources: + limits: + cpu: "4000m" + memory: "8Gi" + requests: + cpu: "200m" + memory: "200Mi" + labels: + - docker + - ubuntu + - jammy + - ubuntu-latest # overlaps w/ hosted runners on GHEC, can use in GHES and GHAE + securityContext: + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: ubuntu-jammy-autoscaling + namespace: runners +spec: + scaleTargetRef: + name: ubuntu-jammy + minReplicas: 1 + maxReplicas: 3 + scaleDownDelaySecondsAfterScaleOut: 60 + metrics: + - type: PercentageRunnersBusy + scaleUpThreshold: "0.75" + scaleDownThreshold: "0.3" + scaleUpFactor: "1.5" + scaleDownFactor: "0.7" diff --git a/deployments/test-ubuntu-jammy.yml b/deployments/test-ubuntu-jammy.yml new file mode 100644 index 0000000..bea7d4f --- /dev/null +++ b/deployments/test-ubuntu-jammy.yml @@ -0,0 +1,34 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: test-ubuntu-jammy + namespace: test-runners +spec: + replicas: 1 + template: + spec: + repository: some-natalie/kubernoodles + env: + - name: DISABLE_RUNNER_UPDATE # Disables automatic runner updates + value: "true" + ephemeral: true + image: ghcr.io/some-natalie/kubernoodles/ubuntu-jammy:latest + imagePullPolicy: Always + imagePullSecrets: + - name: ghcr + dockerdWithinRunnerContainer: true + dockerMTU: 1450 + resources: + limits: + cpu: "1000m" + memory: "2Gi" + requests: + cpu: "200m" + memory: "200Mi" + labels: + - test-ubuntu-jammy + securityContext: + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true diff --git a/deployments/ubuntu-jammy.yml b/deployments/ubuntu-jammy.yml new file mode 100644 index 0000000..9dfaf9e --- /dev/null +++ b/deployments/ubuntu-jammy.yml @@ -0,0 +1,55 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: ubuntu-jammy + namespace: runners +spec: + replicas: 2 + template: + spec: + repository: some-natalie/kubernoodles + env: + - name: DISABLE_RUNNER_UPDATE # Disables automatic runner updates + value: "true" + ephemeral: true + image: ghcr.io/some-natalie/kubernoodles/ubuntu-jammy:latest # change this to the version you really want! + imagePullPolicy: Always + imagePullSecrets: + - name: ghcr + dockerdWithinRunnerContainer: true + dockerMTU: 1450 + resources: + limits: + cpu: "4000m" + memory: "8Gi" + requests: + cpu: "200m" + memory: "200Mi" + labels: + - docker + - ubuntu + - jammy + # - ubuntu-latest # overlaps w/ hosted runners on GHEC, can use in GHES and GHAE + securityContext: + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: ubuntu-jammy-autoscaling + namespace: runners +spec: + scaleTargetRef: + name: ubuntu-jammy + minReplicas: 2 + maxReplicas: 6 + scaleDownDelaySecondsAfterScaleOut: 60 + metrics: + - type: PercentageRunnersBusy + scaleUpThreshold: "0.75" + scaleDownThreshold: "0.3" + scaleUpFactor: "1.5" + scaleDownFactor: "0.7" diff --git a/images/ubuntu-jammy.Dockerfile b/images/ubuntu-jammy.Dockerfile new file mode 100644 index 0000000..32233bc --- /dev/null +++ b/images/ubuntu-jammy.Dockerfile @@ -0,0 +1,141 @@ +FROM ubuntu:22.04 + +# Target architecture +ARG TARGETPLATFORM=linux/amd64 + +# GitHub runner arguments +ARG RUNNER_VERSION=2.294.0 + +# Docker and Docker Compose arguments +ARG DOCKER_CHANNEL=stable +ARG DOCKER_VERSION=20.10.17 +ARG COMPOSE_VERSION=v2.6.0 + +# Other arguments +ARG DEBUG=false + +# Label all the things!! +LABEL \ + org.opencontainers.image.source https://github.com/some-natalie/kubernoodles \ + org.opencontainers.image.title ubuntu-jammy-runner \ + org.opencontainers.image.description "An Ubuntu Jammy (22.04 LTS) based runner image for GitHub Actions" \ + org.opencontainers.image.authors "Natalie Somersall (@some-natalie)" \ + org.opencontainers.image.licenses=MIT \ + org.opencontainers.image.documentation https://github.com/some-natalie/kubernoodles/README.md + +# Set environment variables needed at build +ENV DEBIAN_FRONTEND=noninteractive + +# Copy in environment variables not needed at build +COPY images/.env /.env + +# Shell setup +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Install base software +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + apt-utils \ + ca-certificates \ + curl \ + gcc \ + git \ + iptables \ + libyaml-dev \ + locales \ + lsb-release \ + pkg-config \ + software-properties-common \ + sudo \ + supervisor \ + time \ + tzdata \ + unzip \ + wget \ + zip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Runner user +RUN adduser --disabled-password --gecos "" --uid 1000 runner \ + && groupadd docker \ + && usermod -aG sudo runner \ + && usermod -aG docker runner \ + && echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers + +# Install GitHub CLI +COPY images/software/gh-cli.sh /gh-cli.sh +RUN bash /gh-cli.sh && rm /gh-cli.sh + +# Install kubectl +COPY images/software/kubectl.sh /kubectl.sh +RUN bash /kubectl.sh && rm /kubectl.sh + +RUN test -n "$TARGETPLATFORM" || (echo "TARGETPLATFORM must be set" && false) + +# Docker installation +RUN ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \ + && export ARCH \ + && if [ "$ARCH" = "arm64" ]; then export ARCH=aarch64 ; fi \ + && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \ + && if ! curl -L -o docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz"; then \ + echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${ARCH}'"; \ + exit 1; \ + fi; \ + echo "Downloaded Docker from https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${ARCH}/docker-${DOCKER_VERSION}.tgz"; \ + tar --extract \ + --file docker.tgz \ + --strip-components 1 \ + --directory /usr/local/bin/ ; \ + rm docker.tgz; \ + dockerd --version; \ + docker --version + +# Docker-compose installation +RUN curl -L "https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-Linux-x86_64" -o /usr/local/bin/docker-compose ; \ + chmod +x /usr/local/bin/docker-compose ; \ + docker-compose --version + +ENV RUNNER_ASSETS_DIR=/runnertmp + +# Runner download supports amd64 as x64 +RUN ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \ + && export ARCH \ + && if [ "$ARCH" = "amd64" ]; then export ARCH=x64 ; fi \ + && mkdir -p "$RUNNER_ASSETS_DIR" \ + && cd "$RUNNER_ASSETS_DIR" \ + && curl -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${ARCH}-${RUNNER_VERSION}.tar.gz \ + && tar xzf ./runner.tar.gz \ + && rm runner.tar.gz \ + && ./bin/installdependencies.sh \ + && apt-get autoclean \ + && apt-get autoremove + +RUN echo AGENT_TOOLSDIRECTORY=/opt/hostedtoolcache > /runner.env \ + && mkdir /opt/hostedtoolcache \ + && chgrp runner /opt/hostedtoolcache \ + && chmod g+rwx /opt/hostedtoolcache + +RUN ARCH=$(echo ${TARGETPLATFORM} | cut -d / -f2) \ + && export ARCH \ + && if [ "$ARCH" = "amd64" ]; then export ARCH=x86_64 ; fi \ + && curl -L -o /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.5/dumb-init_1.2.5_${ARCH} \ + && chmod +x /usr/local/bin/dumb-init + +COPY images/modprobe.sh /usr/local/bin/modprobe +COPY images/startup.sh /usr/local/bin/ +COPY images/supervisor/ /etc/supervisor/conf.d/ +COPY images/logger.sh /opt/bash-utils/logger.sh +COPY images/entrypoint.sh /usr/local/bin/ +COPY images/docker/daemon.json /etc/docker/daemon.json + +RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/entrypoint.sh /usr/local/bin/modprobe + +VOLUME /var/lib/docker + +# No group definition, as that makes it harder to run docker. +USER runner + +ENTRYPOINT ["/usr/local/bin/dumb-init", "--"] +CMD ["startup.sh"]