User Workspaces in Kubernetes

Dockerfile

@@ -52,4 +52,5 @@ EXPOSE 22
 EXPOSE 80
 EXPOSE 443
 WORKDIR /opt/pwn.college
-CMD ["dojo", "start"]
+ENTRYPOINT ["dojo"]
+CMD ["start"]

challenge/Dockerfile

@@ -52,6 +52,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
         netcat-openbsd
         socat
         sudo
+        tini
         vim
         wget
         unzip
@@ -566,9 +567,8 @@ FROM builder-tools-pip-${INSTALL_TOOLS_PIP} as builder-tools-pip
 FROM builder-essentials as builder-pwn.college
 
 RUN mkdir /opt/pwn.college
-COPY docker-initialize.sh /opt/pwn.college/docker-initialize.sh
-COPY docker-entrypoint.d /opt/pwn.college/docker-entrypoint.d
-COPY docker-entrypoint.sh /opt/pwn.college/docker-entrypoint.sh
+COPY entrypoint.d /opt/pwn.college/entrypoint.d
+COPY entrypoint.sh /opt/pwn.college/entrypoint.sh
 COPY services.d /opt/pwn.college/services.d
 COPY setuid_interpreter.c /opt/pwn.college/setuid_interpreter.c
 COPY bash.bashrc /opt/pwn.college/bash.bashrc
@@ -849,5 +849,4 @@ RUN <<EOF
     date > /opt/pwn.college/build
 EOF
 
-USER hacker
-WORKDIR /home/hacker
+ENTRYPOINT [ "/opt/pwn.college/entrypoint.sh" ]

challenge/docker-initialize.sh → challenge/entrypoint.d/00_root_challenge_init.sh

@@ -1,8 +1,8 @@
-#!/bin/sh
-
 chown hacker:hacker /home/hacker
 chmod 755 /home/hacker
 
 if [ -x "/challenge/.init" ]; then
     /challenge/.init
 fi
+
+touch /opt/pwn.college/.initialized

challenge/entrypoint.sh

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+
+mkdir -p /run/dojo
+exec >/run/dojo/entrypoint.log 2>&1
+
+for var in $(env | grep -o '^KUBERNETES[^=]*')
+do
+    unset "$var"
+done
+
+echo "$DOJO_FLAG" > /flag
+unset DOJO_FLAG
+
+for script in /opt/pwn.college/entrypoint.d/*
+do
+	user=$(basename "$script" | cut -d_ -f2)
+	echo "[*] running entrypoint script '$script' as user '$user'"
+	su "$user" -c "$script"
+done
+
+exec /bin/tini -- /bin/sleep 6h

challenge/services.d/desktop

@@ -5,9 +5,8 @@ exec 2> /tmp/.dojo/service-desktop.log
 
 mkdir -p /tmp/.dojo/vnc /home/hacker/.vnc
 
-container_id="$(cat /.authtoken)"
-password_interact="$(printf 'desktop-interact' | openssl dgst -sha256 -hmac "$container_id" | awk '{print $2}' | head -c 8)"
-password_view="$(printf 'desktop-view' | openssl dgst -sha256 -hmac "$container_id" | awk '{print $2}' | head -c 8)"
+password_interact="$(printf 'desktop-interact' | openssl dgst -sha256 -hmac "$DOJO_AUTH_TOKEN" | awk '{print $2}' | head -c 8)"
+password_view="$(printf 'desktop-view' | openssl dgst -sha256 -hmac "$DOJO_AUTH_TOKEN" | awk '{print $2}' | head -c 8)"
 printf '%s\n%s\n' "$password_interact" "$password_view" | tigervncpasswd -f > /tmp/.dojo/vnc/passwd
 
 start-stop-daemon --start \
@@ -36,7 +35,7 @@ start-stop-daemon --start \
                   --startas /usr/bin/websockify \
                   -- \
                   --web /usr/share/novnc/ \
-                  dojo-user:6081 \
+                  0.0.0.0:6081 \
                   --unix-target=/tmp/.dojo/vnc/socket \
                   </dev/null \
                   >>/tmp/.dojo/vnc/websockify.log \

challenge/services.d/vscode

@@ -12,7 +12,7 @@ start-stop-daemon --start \
                   --startas /usr/bin/code-server \
                   -- \
                   --auth=none \
-                  --bind-addr=dojo-user:6080 \
+                  --bind-addr=0.0.0.0:6080 \
                   --extensions-dir=/opt/code-server/extensions \
                   --disable-telemetry \
                   </dev/null \

ctfd/requirements.txt

@@ -1,5 +1,6 @@
 # DOJO
 docker==6.1.2
+kubernetes==28.1.0
 pyyaml==5.4.1
 schema==0.7.5
 bleach==6.1.0

docker-compose.yml

@@ -1,6 +1,72 @@
-version: '3.4'
-
 services:
+  kube-server:
+    container_name: kube-server
+    hostname: kube-server
+    image: rancher/k3s:latest
+    command: server --node-taint node-role.kubernetes.io/control-plane:NoSchedule
+    tmpfs:
+      - /run
+      - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+      - K3S_TOKEN=${K3S_TOKEN:?err}
+      - K3S_KUBECONFIG_OUTPUT=/output/kube.yaml
+      - K3S_KUBECONFIG_MODE=644
+    volumes:
+      - ./data/kube/kubeconfig:/output
+      - ./data/kube/node/server:/etc/rancher/node
+      - ./data/kube/server:/var/lib/rancher/k3s
+      - ./kube/k3s:/etc/rancher/k3s:ro
+      - ./kube/manifests:/var/lib/rancher/k3s/server/manifests/dojo:ro
+      - ./kube/bin:/usr/local/bin:ro
+    ports:
+      - 6443:6443  # Kubernetes API Server
+      # - 80:80      # Ingress controller port 80
+      # - 443:443    # Ingress controller port 443
+    extra_hosts:
+      - homes-nfs:10.43.0.20
+
+  kube-agent:
+    container_name: kube-agent
+    hostname: kube-agent
+    image: rancher/k3s:latest
+    tmpfs:
+      - /run
+      - /var/run
+    ulimits:
+      nproc: 65535
+      nofile:
+        soft: 65535
+        hard: 65535
+    privileged: true
+    restart: always
+    environment:
+      - K3S_URL=https://kube-server:6443
+      - K3S_TOKEN=${K3S_TOKEN:?err}
+    volumes:
+      - ./data/kube/node/agent:/etc/rancher/node
+      - ./data/kube/agent:/var/lib/rancher/k3s
+      - ./data/homes:/var/homes:shared
+      - ./kube/k3s:/etc/rancher/k3s:ro
+      - ./kube/bin:/usr/local/bin:ro
+    extra_hosts:
+      - homes-nfs:10.43.0.20
+
+  registry:
+    container_name: registry
+    image: registry:2
+    restart: always
+    ports:
+      - "5000:5000"
+    volumes:
+      - ./data/registry:/var/lib/registry
+
   challenge:
     build:
       context: ./challenge
@@ -30,10 +96,13 @@ services:
         - UBUNTU_VERSION=${UBUNTU_VERSION}
         - DOJO_CHALLENGE=${DOJO_CHALLENGE}
       privileged: true
+    image: localhost:5000/challenge
     platform: linux/amd64
     entrypoint: /bin/true
-    networks:
-      - user_network
+
+  volume_nfs:
+    build: ./volume_nfs
+    image: localhost:5000/volume-nfs
 
   ctfd:
     container_name: ctfd
@@ -70,6 +139,7 @@ services:
       - SECRET_KEY=${SECRET_KEY}
       - DOJO_HOST=${DOJO_HOST}
       - HOST_DATA_PATH=/opt/pwn.college/data
+      - KUBECONFIG=/var/kubeconfig/kube.yaml
       - MAIL_SERVER=${MAIL_SERVER}
       - MAIL_PORT=${MAIL_PORT}
       - MAIL_USERNAME=${MAIL_USERNAME}
@@ -90,6 +160,7 @@ services:
       - ./data/homes:/var/homes:shared
       - ./data/challenges:/var/challenges:ro
       - ./data/dojos:/var/dojos
+      - ./data/kube/kubeconfig:/var/kubeconfig:ro
       - ./data/ssh_host_keys/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro
       - ./index.html:/var/index.html:ro
       - ./user_firewall.allowed:/var/user_firewall.allowed:ro
@@ -149,8 +220,9 @@ services:
       context: ./sshd
     volumes:
       - ./data/ssh_host_keys:/etc/ssh:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - ./data/kube/kubeconfig:/var/kubeconfig:ro
     environment:
+      - KUBECONFIG=/var/kubeconfig/kube.yaml
       - DB_HOST=${DB_HOST}
       - DB_NAME=${DB_NAME}
       - DB_USER=${DB_USER}
@@ -188,12 +260,6 @@ services:
       - ./nginx-proxy/etc/passwd:/etc/passwd:ro
       - ./data/homes:/var/homes:shared
       - /var/run/docker.sock:/tmp/${DOCKER_PSLR}/docker.sock:ro
-    networks:
-      default:
-      user_network:
-        aliases:
-          - nginx
-        ipv4_address: 10.0.0.3
 
   nginx-certs:
     container_name: nginx_certs
@@ -212,14 +278,3 @@ volumes:
   dhparam:
   certs:
   acme:
-
-networks:
-  user_network:
-    name: user_network
-    driver: bridge
-    ipam:
-      config:
-        - subnet: 10.0.0.0/8
-    driver_opts:
-      com.docker.network.bridge.name: "user_network"
-      com.docker.network.bridge.enable_icc: "false"