Merge pull request #223 from yfyf/release-validation-finalizations

Release validation test finalizations

Author: guyonvarch

.github/workflows/test.yml

@@ -44,6 +44,8 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/prep-build-env
+    - name: Make magic-nix-cache read-only by removing post-build-hook
+      run: sed -i '/post-build-hook = magic-nix-cache-build-hook/d' $HOME/.config/nix/nix.conf
     - run: NIXPKGS_ALLOW_UNFREE=1 nix-build -A driverInteractive ${{ matrix.file }}
 
 

Readme.md

@@ -105,11 +105,17 @@ Building a complete system takes time, so it is a good idea to test at the compo
 
 ### Automated Testing
 
-There are 3 types of tests:
+There are 4 types of automated tests:
 
 - unit tests (used in kiosk and controller)
-- subcomponent integration tests (see below)
-- end-to-end system level tests (see below)
+- subcomponent integration tests
+- end-to-end system level tests
+- release validation tests
+
+All but the last are run on each commit via Github Actions.
+
+See subsections below for more details on each test type.
+
 
 #### Subcomponent (integration) tests
 
@@ -150,6 +156,35 @@ run 10x slower. If you see `failed to initialize KVM` in the console logs, it
 means there's a problem. See [this (outdated) Github issue](https://github.com/NixOS/nixpkgs/issues/124371#issue-900719073)
 for more details.
 
+
+#### Release validation (self-update) test
+
+The release validation test is used to perform the final _automated_ checks before
+manually testing and officially publishing a release.
+
+Currently it tests only one critical path: the self-update scenario from an
+earlier release (the 'base' system) to the current/upcoming release (the 'next'
+system).
+
+The test does not alter the configuration of the base or next systems' in any way
+(e.g. no test-instrumentation.nix extras, which are present in end-to-end
+tests).
+
+To run the tests against the previous release, execute:
+
+    nix-build testing/release-validation.nix
+
+To test against a specific base system (e.g. `2023.9.1-DISK`), add the flag
+`--arg baseSystemVersion '"2023.9.1-DISK"'`.
+
+Compressed base system disk images are created for every tagged release using
+`./build release-disk` and the [release](.github/workflows/release-tag.yml)
+Github workflow. See the "Test disk" links in [release
+summaries](https://github.com/dividat/playos/releases).
+
+For more details, see the comments in
+[testing/release-validation.nix](testing/release-validation.nix).
+
 ## Deployment
 
 Update bundles are hosted on Amazon S3. The script `bin/deploy-update` will handle signing and uploading of bundle.

testing/release-validation.nix

@@ -24,13 +24,22 @@
 #   - execute "sendkey ctrl-shift-f1" (switch to TTY1)
 #   - login with "root"
 let
-    baseS3URL = "https://dividat-playos-test-disks.s3.amazonaws.com/by-tag";
+    # Note: we use HTTP instead of HTTPS, because pkgs.fetchurl fails
+    # when __structuredAttrs is enabled (due to mysterious OpenSSL/TLS errors)
+    # and also fails when __structuredAttrs is disabled (due to
+    # https://github.com/NixOS/nixpkgs/issues/177660).
+    # HTTP usage is fine since the output hash is fixed and verifies the download.
+    baseS3URL = "http://dividat-playos-test-disks.s3.amazonaws.com/by-tag";
     # Generated via ./build release-disk and .github/workflows/release-tag.yml
     # See https://github.com/dividat/playos/releases
     diskImageURLs = {
-        "1.0.0-TEST" = {
-            url = "${baseS3URL}/playos-disk-1.0.0-TEST.img.zst";
-            hash = "sha256-7cyStGfsxVyQ2ugkI9XRFnNrnPhd5QRf+oAQxLu3ovM=";
+        "2023.9.1-DISK" = {
+            url = "${baseS3URL}/playos-release-disk-2023.9.1-DISK.img.zst";
+            hash = "sha256-eTyNcDkYSsMUsUHToZJ4tEKag9WSi8gA2SAihFFCqH0=";
+        };
+        "2024.7.0-DISK" = {
+            url = "${baseS3URL}/playos-release-disk-2024.7.0-DISK.img.zst";
+            hash = "sha256-vJDB99ICt0W1PmONikNY5wwIF7oQU388DzYRgPqkooY=";
         };
     };
 in
@@ -48,9 +57,9 @@ in
     kioskUrlDomain ? "kiosk-server.local",
 
     # PlayOS system we are updating from
-    baseSystemVersion ? "1.0.0-TEST",
+    baseSystemVersion ? "2024.7.0-DISK",
 
-    # A downloadable URL containing a zstd compressed disk image
+    # A zstd-compressed PlayOS disk image
     baseSystemDiskImage ? (pkgs.fetchurl diskImageURLs.${baseSystemVersion})
         .overrideAttrs {
             __structuredAttrs = true;
@@ -150,20 +159,23 @@ pkgs.testers.runNixOSTest {
     "-monitor" "vc"
   ];
 
-  enableOCR = true;
-
   extraPythonPackages = ps: [
     ps.colorama
     ps.types-colorama
     ps.requests
     ps.types-requests
     ps.tesserocr
+    ps.pillow
+    ps.types-pillow
   ];
 
   testScript = {nodes}: ''
 ${builtins.readFile ./helpers/nixos-test-script-helpers.py}
 ${builtins.readFile ./end-to-end/tests/base/proxy-and-update-helpers.py}
 import tesserocr # type: ignore
+import PIL.Image
+import PIL.ImageEnhance
+import PIL.ImageOps
 import tempfile
 import time
 import atexit
@@ -191,13 +203,17 @@ extract_base_system_disk("${baseSystemDiskImage}", "${overlayPath}")
 playos.start(allow_reboot=True)
 sidekick.start()
 
-# Less accurate, but much faster OCR than NixOS `get_screen_text`,
-# which takes almost 20 seconds per call.
-# Fails to identify white text on dark backgrounds.
+# Faster OCR than NixOS `get_screen_text`, which takes almost 20 seconds per
+# call. Fails to identify white text on dark backgrounds.
 def screenshot_and_ocr(vm):
     with tempfile.TemporaryDirectory() as d:
         vm.screenshot(d + "/screenshot.png")
-        return tesserocr.file_to_text(d + "/screenshot.png")
+        im = PIL.Image.open(d + "/screenshot.png")
+        im = PIL.ImageOps.grayscale(im)
+        im = PIL.ImageEnhance.Brightness(im).enhance(1.5)
+        im = PIL.ImageEnhance.Contrast(im).enhance(4.0)
+        return tesserocr.image_to_text(im)
+
 
 def wait_until_passes(test, retries=10, sleep=1):
     while True: