Spread test for multiple distros (#4211)

.github/workflows/build-snap.yml

@@ -197,6 +197,33 @@ jobs:
         run: |
           sudo -E bash -x -c "./tests/libs/airgap.sh --distro ubuntu:20.04 --channel $PWD/build/microk8s.snap"
 
+  test-spread:
+    name: Test microk8s on multi distros
+    runs-on: ubuntu-20.04
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        distro: ["images:centos/7", "images:debian/12"]
+    steps:
+      - name: Checking out repo
+        uses: actions/checkout@v4
+      - name: Fetch snap
+        uses: actions/[email protected]
+        with:
+          name: microk8s.snap
+          path: build
+      - name: Initialize LXD
+        run: |
+          sudo apt install apparmor apparmor-utils -y
+          sudo lxd init --auto
+          sudo lxc network set lxdbr0 ipv6.address=none
+          sudo usermod --append --groups lxd $USER
+          sg lxd -c 'lxc version'
+      - name: Run spread tests
+        run: |
+          sudo -E bash -x -c "./tests/libs/spread.sh --distro ${{ matrix.distro }} --channel $PWD/build/microk8s.snap"
+
   security-scan:
     name: Security scan
     runs-on: ubuntu-20.04

tests/libs/spread.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -ex
+
+source tests/libs/utils.sh
+
+function run_spread_tests() {
+  local NAME=$1
+  local DISTRO=$2
+  local PROXY=$3
+  local TO_CHANNEL=$4
+
+  create_machine "$NAME" "$DISTRO" "$PROXY"
+
+  if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]]
+  then
+    lxc file push "${TO_CHANNEL}" "$NAME"/tmp/microk8s_latest_amd64.snap
+    for i in {1..5}; do lxc exec "$NAME" -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic && break || sleep 5; done
+  else
+    lxc exec "$NAME" -- snap install microk8s --channel="${TO_CHANNEL}" --classic
+  fi
+
+  lxc exec "$NAME" -- /snap/bin/microk8s stop
+  lxc exec "$NAME" -- sed -i '/\[plugins."io.containerd.grpc.v1.cri"\]/a \ \ disable_apparmor=true' /var/snap/microk8s/current/args/containerd-template.toml
+  lxc exec "$NAME" -- /snap/bin/microk8s start
+  lxc exec "$NAME" -- /snap/bin/microk8s status --wait-ready --timeout 300
+  sleep 45
+  lxc exec "$NAME" -- /snap/bin/microk8s kubectl wait pod --all --for=condition=Ready -A --timeout=300s
+  lxc exec "$NAME" -- script -e -c "pytest -s /root/tests/test-simple.py"
+}
+
+TEMP=$(getopt -o "l,h" \
+              --long help,lib-mode,node-name:,distro:,channel:,proxy: \
+              -n "$(basename "$0")" -- "$@")
+
+if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi
+
+eval set -- "$TEMP"
+
+NAME="${NAME-"machine-$RANDOM"}"
+DISTRO="${DISTRO-}"
+TO_CHANNEL="${TO_CHANNEL-}"
+PROXY="${PROXY-}"
+LIBRARY_MODE=false
+
+while true; do
+  case "$1" in
+    -l | --lib-mode ) LIBRARY_MODE=true; shift ;;
+    --node-name ) NAME="$2"; shift 2 ;;
+    --distro ) DISTRO="$2"; shift 2 ;;
+    --channel ) TO_CHANNEL="$2"; shift 2 ;;
+    --proxy ) PROXY="$2"; shift 2 ;;
+    -h | --help )
+      prog=$(basename -s.wrapper "$0")
+      echo "Usage: $prog [options...]"
+      echo "     --node-name <name> Name to be used for LXD containers"
+      echo "         Can also be set by using NAME environment variable"
+      echo "     --distro <distro> Distro image to be used for LXD containers Eg. ubuntu:18.04"
+      echo "         Can also be set by using DISTRO environment variable"
+      echo "     --channel <channel> Channel to be tested Eg. latest/edge"
+      echo "         Can also be set by using TO_CHANNEL environment variable"
+      echo "     --proxy <url> Proxy url to be used by the nodes"
+      echo "         Can also be set by using PROXY environment variable"
+      echo " -l, --lib-mode Make the script act like a library Eg. true / false"
+      echo
+      exit ;;
+    -- ) shift; break ;;
+    * ) break ;;
+  esac
+done
+
+if [ "$LIBRARY_MODE" == "false" ];
+then
+  run_spread_tests "$NAME" "$DISTRO" "$PROXY" "$TO_CHANNEL"
+fi

tests/libs/utils.sh

@@ -14,6 +14,8 @@ function create_machine() {
 
   # Allow for the machine to boot and get an IP
   sleep 20
+  # CentOS 8,9 variants(rocky, alma) don't ship with tar, such a dirty hack...
+  lxc exec "$NAME" -- /bin/bash -c "yum install tar -y || true"
   tar cf - ./tests | lxc exec "$NAME" -- tar xvf - -C /root
   DISTRO_DEPS_TMP="${DISTRO//:/_}"
   DISTRO_DEPS="${DISTRO_DEPS_TMP////-}"
@@ -22,7 +24,7 @@ function create_machine() {
   sleep 20
 
   trap 'lxc delete '"${NAME}"' --force || true' EXIT
-  if [ "$#" -ne 1 ]
+  if [ ! -z "${PROXY}" ]
   then
     lxc exec "$NAME" -- /bin/bash -c "echo HTTPS_PROXY=$PROXY >> /etc/environment"
     lxc exec "$NAME" -- /bin/bash -c "echo https_proxy=$PROXY >> /etc/environment"

tests/lxc/install-deps/images_almalinux-8

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+dnf install epel-release -y
+dnf upgrade -y
+yum install sudo -y
+yum install fuse squashfuse -y
+yum install snapd -y
+systemctl enable --now snapd.socket
+ln -s /var/lib/snapd/snap /snap
+yum install python3-pip  -y
+yum install docker -y
+pip3 install pytest requests pyyaml sh
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_centos-8-Stream

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+dnf install epel-release -y
+dnf upgrade -y
+yum install sudo -y
+yum install fuse squashfuse -y
+yum install snapd -y
+systemctl enable --now snapd.socket
+ln -s /var/lib/snapd/snap /snap
+yum install python3-pip  -y
+yum install docker -y
+pip3 install pytest requests pyyaml sh
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_debian-10

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+export $(grep -v '^#' /etc/environment | xargs)
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y
+pip3 install pytest requests pyyaml sh
+# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698
+# if core is to be installed by microk8s it fails
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done
+
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_debian-11

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+export $(grep -v '^#' /etc/environment | xargs)
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y
+pip3 install pytest requests pyyaml sh
+# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698
+# if core is to be installed by microk8s it fails
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done
+
+
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_debian-12

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+export $(grep -v '^#' /etc/environment | xargs)
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get update
+apt-get install python3-pip docker.io libsquashfuse0 squashfuse fuse snapd -y
+pip3 install pytest requests pyyaml sh --break-system-packages
+# Attempting to address https://forum.snapcraft.io/t/lxd-refresh-cause-container-socket-error/8698
+# if core is to be installed by microk8s it fails
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_fedora-37

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+yum install epel-release -y
+yum install sudo -y
+yum install fuse squashfuse -y
+yum install snapd -y
+systemctl enable --now snapd.socket
+ln -s /var/lib/snapd/snap /snap
+yum install python3-pip  -y
+yum install docker -y
+pip3 install pytest requests pyyaml sh
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_fedora-38

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+yum install epel-release -y
+yum install sudo -y
+yum install fuse squashfuse -y
+yum install snapd -y
+systemctl enable --now snapd.socket
+ln -s /var/lib/snapd/snap /snap
+yum install python3-pip  -y
+yum install docker -y
+pip3 install pytest requests pyyaml sh
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/lxc/install-deps/images_rockylinux-8

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+dnf install epel-release -y
+dnf upgrade -y
+yum install sudo -y
+yum install fuse squashfuse -y
+yum install snapd -y
+systemctl enable --now snapd.socket
+ln -s /var/lib/snapd/snap /snap
+yum install python3-pip  -y
+yum install docker -y
+pip3 install pytest requests pyyaml sh
+
+# wait for the snapd seeding to take place!
+n=0
+until [ $n -ge 7 ]
+do
+  sudo snap install core20 && break  # substitute your command here
+  n=$[$n+1]
+  sleep 10
+done

tests/test-simple.py

@@ -76,11 +76,11 @@ def test_nginx_ingress(self):
                     "jsonpath={.spec.clusterIP}",
                 ]
             )
-            response = requests.get(f"http://{output.decode('utf-8')}:80")
+            response = requests.get(f"http://{output.decode('utf-8')}:80", timeout=15)
         else:
             # Wait for ingress to be ready
             time.sleep(3)
-            response = requests.get("http://127.0.0.1:80")
+            response = requests.get("http://127.0.0.1:80", timeout=15)
 
         subprocess.run(
             ["microk8s", "kubectl", "delete", "-f", "tests/templates/simple-deploy.yaml"],