Add webcam streaming (#34)

* Add webcam streaming * test: webcam streaming * feat: remove webcam local recording * feat: reduce webcam streaming resolution to 720p * fix: ffmpeg record use port 101 from vlc * fix(dev_env): use xhci (usb 3.0) * fix: handle device disconnect * fix: use udevadm to monitor webcam events * fix(startup.sh): vlc spelling * feat: include input audio in webcam streaming * fix: Set webcam keyframe interval to 15 fps Using fixed keyframe interval significantly reduce stream latency and reduce *.ts fragment file size. * fix(startup.sh): move udevadm up before vlc to avoid missing video device remove event * feat(startup.sh): wait kills udevadm and cvlc only if terminated instead of status change * feat(startup.sh): udevadm specify video device * feat(startup.sh): prevent cpu hogging when video device not present and print debug * feat(startup.sh): kill -9 if cvlc fails to exit * fix(startup.sh): let kill terminate with exit code because we're not using exit on error * feat(startup.sh): pick webcams by id rather than /dev/video0 * feat: process substitution instead of pipeline for udevadm * fix(startup.sh): correct timeout * feat: simplify process killing logic * feat: add sigkill to startup.sh's peer stop * feat: Add audio device picker * feat: Add audio device utils for vnoiconf.sh * feat: allow glob and manual audio source config * feat: add comments to regexes in vnoiconf.sh * feat: pattern matching notes * feat: move default audio_source for clarity * chore: Cleanup github workflow jobs * fix: fix workflow image-toolkit config secret name * chore: drop root requirement when generating secrets * feat: Use workaround to gain extra space from github actions * fix: keep docker cache and image * feat: Remove IMAGE and ICPC folder in github actions mode * fix: Reorder github actions remove files * fix: use different tmp path for gh workflows scp actions * fix: Use homemade scp script * fix: please fix this scp madness --------- Co-authored-by: Nguyen Minh Nhat Noe <[email protected]> Co-authored-by: Minh Hung Tran <[email protected]>
VNOI-Admin · Jul 11, 2024 · 7c84b51 · 7c84b51
1 parent 31f7be4
commit 7c84b51
Show file tree

Hide file tree

Showing 9 changed files with 306 additions and 31 deletions.
diff --git a/.github/workflows/push-releases.yml b/.github/workflows/push-releases.yml
@@ -19,6 +19,20 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 512
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-docker-images: 'true'
+          remove-codeql: 'true'
+          remove-haskell: 'true'
+          remove-android: 'true'
+
+      - name: Check free space
+        run: echo "Free space $(df -h /)"
+
       - name: Checkout Repository
         uses: actions/checkout@v3
 
@@ -34,16 +48,10 @@ jobs:
           CONTENT: ${{ secrets.CONFIG_LOCAL_SH }}
           FILENAME: config.local.sh
 
-      - name: Write image-toolkit/config.local.sh
-        run: echo $CONTENT | base64 -d > $FILENAME
-        env:
-          CONTENT: ${{ secrets.SRC_CONFIG_LOCAL_SH }}
-          FILENAME: image-toolkit/config.local.sh
-
       - name: Write image-toolkit/config.sh
         run: echo $CONTENT | base64 -d > $FILENAME
         env:
-          CONTENT: ${{ secrets.SRC_CONFIG_SH }}
+          CONTENT: ${{ secrets.TOOLKIT_CONFIG_SH }}
           FILENAME: image-toolkit/config.sh
 
       - name: Build image
@@ -58,7 +66,7 @@ jobs:
           release_name: ${{ env.image_name }}
           draft: false
           prerelease: false
-      
+
       - name: Split ISO
         run: mkdir iso-parts; split -b 2000MB live-build/contestant.iso iso-parts/contestant
 

diff --git a/.github/workflows/push-self-hosted-runner.yml b/.github/workflows/push-self-hosted-runner.yml
@@ -17,6 +17,20 @@ jobs:
     runs-on: self-hosted
 
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 512
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-docker-images: 'true'
+          remove-codeql: 'true'
+          remove-haskell: 'true'
+          remove-android: 'true'
+
+      - name: Check free space
+        run: echo "Free space $(df -h /)"
+
       - name: Checkout Repository
         uses: actions/checkout@v3
 
@@ -32,16 +46,10 @@ jobs:
           CONTENT: ${{ secrets.CONFIG_LOCAL_SH }}
           FILENAME: config.local.sh
 
-      - name: Write image-toolkit/config.local.sh
-        run: echo $CONTENT | base64 -d > $FILENAME
-        env:
-          CONTENT: ${{ secrets.SRC_CONFIG_LOCAL_SH }}
-          FILENAME: image-toolkit/config.local.sh
-
       - name: Write image-toolkit/config.sh
         run: echo $CONTENT | base64 -d > $FILENAME
         env:
-          CONTENT: ${{ secrets.SRC_CONFIG_SH }}
+          CONTENT: ${{ secrets.TOOLKIT_CONFIG_SH }}
           FILENAME: image-toolkit/config.sh
 
       - name: Build image

diff --git a/.github/workflows/push-self-hosted-storage.yml b/.github/workflows/push-self-hosted-storage.yml
@@ -14,6 +14,19 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 512
+          swap-size-mb: 1024
+          remove-dotnet: 'true'
+          remove-codeql: 'true'
+          remove-haskell: 'true'
+          remove-android: 'true'
+
+      - name: Check free space
+        run: echo "Free space $(df -h /)"
+
       - name: Checkout Repository
         uses: actions/checkout@v3
 
@@ -29,28 +42,32 @@ jobs:
           CONTENT: ${{ secrets.CONFIG_LOCAL_SH }}
           FILENAME: config.local.sh
 
-      - name: Write image-toolkit/config.local.sh
-        run: echo $CONTENT | base64 -d > $FILENAME
-        env:
-          CONTENT: ${{ secrets.SRC_CONFIG_LOCAL_SH }}
-          FILENAME: image-toolkit/config.local.sh
-
       - name: Write image-toolkit/config.sh
         run: echo $CONTENT | base64 -d > $FILENAME
         env:
-          CONTENT: ${{ secrets.SRC_CONFIG_SH }}
+          CONTENT: ${{ secrets.TOOLKIT_CONFIG_SH }}
           FILENAME: image-toolkit/config.sh
 
       - name: Build image
         run: sudo ./build.sh icpc_build --github-actions
 
+      # - name: Push to Server
+      #   uses: appleboy/[email protected]
+      #   with:
+      #     host: ${{ secrets.SCP_HOST }}
+      #     username: ${{ secrets.SCP_USERNAME }}
+      #     key: ${{ secrets.SCP_KEY }}
+      #     port: ${{ secrets.SCP_PORT }}
+      #     source: live-build/contestant.iso
+      #     target: ${{ secrets.SCP_TARGET }}
+      #     tar_tmp_path: live-build/
+
       - name: Push to Server
-        if: ${{ github.event.inputs.iso_location == 'server' }}
-        uses: appleboy/[email protected]
+        uses: nogsantos/scp-deploy@master
         with:
+          src: live-build/contestant.iso
           host: ${{ secrets.SCP_HOST }}
-          username: ${{ secrets.SCP_USERNAME }}
-          key: ${{ secrets.SCP_KEY }}
+          remote: ${{ secrets.SCP_TARGET }}
           port: ${{ secrets.SCP_PORT }}
-          source: live-build/contestant.iso
-          target: ${{ secrets.SCP_TARGET }}
+          user: ${{ secrets.SCP_USERNAME }}
+          key: ${{ secrets.SCP_KEY }}
diff --git a/build.sh b/build.sh
@@ -339,6 +339,13 @@ icpc_image_build() {
             -no-emul-boot \
         -o "../$IMAGE_FILENAME" \
         .
+
+    if [ $CLEAR_EARLY = true ]; then
+        log "Clearing early to free up space"
+        rm -rf $IMAGE
+        log "Done"
+    fi
+
     log "Build finished. Cleaning up (run clean command for full clean up)."
 }
 
@@ -526,7 +533,8 @@ dev_create() {
     vboxmanage modifyvm "$VM_NAME" \
         --memory $MEM \
         --cpus $CPUS \
-        --firmware $FIRMWARE
+        --firmware $FIRMWARE \
+        --usb-xhci=on
     log "Done"
 
     log "Creating disk"
@@ -620,7 +628,6 @@ case $1 in
         icpc_build $@
         ;;
     generate_actions_secret)
-        assert_root
         generate_actions_secret
         ;;
     dev_reload)

diff --git a/image-toolkit/bin/vnoiconf.sh b/image-toolkit/bin/vnoiconf.sh
@@ -14,6 +14,26 @@ check_ip()
 	fi
 }
 
+check_audio_device() {
+	if [[ "$1" = "any" ]] || [[ "$1" = "system" ]]; then
+		return 0
+	fi
+
+	local AUDIO_DEVICE_INFO
+	mapfile -t AUDIO_DEVICE_INFO < <(find /proc/asound/ -regex ".*card[0-9]+/pcm[0-9]+c/info")
+	for info_file in "${AUDIO_DEVICE_INFO[@]}"; do
+		local NAME=$(sed -n "s/^name: //p" "$info_file")
+
+		# Pattern matching is used
+		# https://www.gnu.org/software/bash/manual/html_node/Bash-Conditional-Expressions.html#Bash-Conditional-Expressions
+		if [[ "$NAME" = $1 ]]; then
+			return 0
+		fi
+	done
+
+	return 1
+}
+
 logger -p local0.info "VNOICONF: invoke $1"
 
 case "$1" in
@@ -129,6 +149,33 @@ EOM
 # EOM
 # 		fi
 # 		;;
+	set_audio_device)
+		# Get device name at the first argument and replace the AUDIO_DEVICE_NAME in config.sh with it
+		if [ -z "$2" ]; then
+			echo "No device name specified"
+			exit 1
+		fi
+		if ! check_audio_device "$2"; then
+			echo "Warning: No audio devices match the name $2"
+		fi
+		# Matches whole line starting with "AUDIO_DEVICE_NAME="
+		sed -i "s/AUDIO_DEVICE_NAME=.*/AUDIO_DEVICE_NAME=\"$2\"/" /opt/vnoi/config.sh
+		# Restart stream
+		if [[ -f "/run/icpc-webcam-stream.pid" ]]; then
+			kill -9 $(cat /run/icpc-webcam-stream.pid) 2> /dev/null
+		fi
+		;;
+	list_audio_devices)
+		# "[0-9]+" matches one or more digits
+		mapfile -t AUDIO_DEVICE_INFO < <(find /proc/asound/ -regex ".*card[0-9]+/pcm[0-9]+c/info")
+		for info_file in "${AUDIO_DEVICE_INFO[@]}"; do
+			CARD_NO=$(sed -n "s/^card: //p" "$info_file")
+			DEVICE_NO=$(sed -n "s/^device: //p" "$info_file")
+			NAME=$(sed -n "s/^name: //p" "$info_file")
+
+			echo "$CARD_NO,$DEVICE_NO: $NAME"
+		done
+		;;
 	*)
 		echo Not allowed
 		;;

diff --git a/image-toolkit/config.sh.sample b/image-toolkit/config.sh.sample
@@ -12,3 +12,11 @@ CONTEST_SITE_DOMAIN_NAME=""  # Example: contest.icpc.info
 
 ADMIN_SUBNET="10.1.0.0/16"
 COACH_SUBNET="10.20.0.0/16"
+
+# Name of the audio device used for streaming (Example: "Intel 82801AA-ICH - MIC ADC", can be a glob pattern)
+# "any" will be the first available hardware capture device
+# "system" will be the default device of the system
+AUDIO_DEVICE_NAME="any"
+
+# If AUDIO_DEVICE_SOURCE is provided, AUDIO_DEVICE_NAME check is bypassed and AUDIO_DEVICE_SOURCE will be used directly
+# AUDIO_DEVICE_SOURCE="alsa://plughw:1,0"