From ebf4f19004be4fd0d3e1f5ef32f57e42930b47da Mon Sep 17 00:00:00 2001
From: Disservin <disservin.social@gmail.com>
Date: Tue, 10 Sep 2024 18:36:54 +0200
Subject: [PATCH] Port instrumented testing to python

Since an unknown amount of time the instrumented CI has been a bit
flawed, explained here
https://github.com/official-stockfish/Stockfish/issues/5185. It also
experiences random timeout issues where restarting the workflow fixes it
or very long run times (more than other workflows) and is not very
portable.

The intention of this commit is to port the instrumented.sh to python
which also works on other operating systems. It should also be
relatively easy for beginners to add new tests to assert stockfish's
output and to run it.
From the source directory the following command can be run.

`python3 ../tests/instrumented.py --none ./stockfish`

A test runner will go over the test suites and run the test cases.

All instrumented tests should have been ported over.
The required python version for this is should be 3.7 (untested),
testing.py includes some infrastructure code which setups the testing.

fixes https://github.com/official-stockfish/Stockfish/issues/5185
closes https://github.com/official-stockfish/Stockfish/pull/5583

No functional change
---
 .github/workflows/sanitizers.yml |   2 +-
 .gitignore                       |   5 +
 tests/instrumented.py            | 520 +++++++++++++++++++++++++++++++
 tests/instrumented.sh            | 301 ------------------
 tests/testing.py                 | 372 ++++++++++++++++++++++
 5 files changed, 898 insertions(+), 302 deletions(-)
 create mode 100644 tests/instrumented.py
 delete mode 100755 tests/instrumented.sh
 create mode 100644 tests/testing.py

diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml
index 55459292107..946a81cec4a 100644
--- a/.github/workflows/sanitizers.yml
+++ b/.github/workflows/sanitizers.yml
@@ -75,4 +75,4 @@ jobs:
           export CXXFLAGS="-O1 -fno-inline"
           make clean
           make -j4 ARCH=x86-64-sse41-popcnt ${{ matrix.sanitizers.make_option }} debug=yes optimize=no build > /dev/null
-          ../tests/instrumented.sh --${{ matrix.sanitizers.instrumented_option }}
+          python3 ../tests/instrumented.py --${{ matrix.sanitizers.instrumented_option }} ./stockfish
diff --git a/.gitignore b/.gitignore
index 8981efcaf13..2fc80d48731 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,8 @@ src/-lstdc++.res
 # Neural network for the NNUE evaluation
 **/*.nnue
 
+# Files generated by the instrumented tests
+tsan.supp
+__pycache__/
+tests/syzygy
+tests/bench_tmp.epd
\ No newline at end of file
diff --git a/tests/instrumented.py b/tests/instrumented.py
new file mode 100644
index 00000000000..bb7efa80913
--- /dev/null
+++ b/tests/instrumented.py
@@ -0,0 +1,520 @@
+import argparse
+import re
+import sys
+import subprocess
+import pathlib
+import os
+
+from testing import (
+    EPD,
+    TSAN,
+    Stockfish as Engine,
+    MiniTestFramework,
+    OrderedClassMembers,
+    Valgrind,
+    Syzygy,
+)
+
+PATH = pathlib.Path(__file__).parent.resolve()
+CWD = os.getcwd()
+
+
+def get_prefix():
+    if args.valgrind:
+        return Valgrind.get_valgrind_command()
+    if args.valgrind_thread:
+        return Valgrind.get_valgrind_thread_command()
+
+    return []
+
+
+def get_threads():
+    if args.valgrind_thread or args.sanitizer_thread:
+        return 2
+    return 1
+
+
+def get_path():
+    return os.path.abspath(os.path.join(CWD, args.stockfish_path))
+
+
+def postfix_check(output):
+    if args.sanitizer_undefined:
+        for idx, line in enumerate(output):
+            if "runtime error:" in line:
+                # print next possible 50 lines
+                for i in range(50):
+                    debug_idx = idx + i
+                    if debug_idx < len(output):
+                        print(output[debug_idx])
+                return False
+
+    if args.sanitizer_thread:
+        for idx, line in enumerate(output):
+            if "WARNING: ThreadSanitizer:" in line:
+                # print next possible 50 lines
+                for i in range(50):
+                    debug_idx = idx + i
+                    if debug_idx < len(output):
+                        print(output[debug_idx])
+                return False
+
+    return True
+
+
+def Stockfish(*args, **kwargs):
+    return Engine(get_prefix(), get_path(), *args, **kwargs)
+
+
+class TestCLI(metaclass=OrderedClassMembers):
+
+    def beforeAll(self):
+        pass
+
+    def afterAll(self):
+        pass
+
+    def beforeEach(self):
+        self.stockfish = None
+
+    def afterEach(self):
+        assert postfix_check(self.stockfish.get_output()) == True
+        self.stockfish.clear_output()
+
+    def test_eval(self):
+        self.stockfish = Stockfish("eval".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_nodes_1000(self):
+        self.stockfish = Stockfish("go nodes 1000".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_depth_10(self):
+        self.stockfish = Stockfish("go depth 10".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_perft_4(self):
+        self.stockfish = Stockfish("go perft 4".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_movetime_1000(self):
+        self.stockfish = Stockfish("go movetime 1000".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_wtime_8000_btime_8000_winc_500_binc_500(self):
+        self.stockfish = Stockfish(
+            "go wtime 8000 btime 8000 winc 500 binc 500".split(" "),
+            True,
+        )
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_wtime_1000_btime_1000_winc_0_binc_0(self):
+        self.stockfish = Stockfish(
+            "go wtime 1000 btime 1000 winc 0 binc 0".split(" "),
+            True,
+        )
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_wtime_1000_btime_1000_winc_0_binc_0_movestogo_5(self):
+        self.stockfish = Stockfish(
+            "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5".split(" "),
+            True,
+        )
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_movetime_200(self):
+        self.stockfish = Stockfish("go movetime 200".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_go_nodes_20000_searchmoves_e2e4_d2d4(self):
+        self.stockfish = Stockfish(
+            "go nodes 20000 searchmoves e2e4 d2d4".split(" "), True
+        )
+        assert self.stockfish.process.returncode == 0
+
+    def test_bench_128_threads_8_default_depth(self):
+        self.stockfish = Stockfish(
+            f"bench 128 {get_threads()} 8 default depth".split(" "),
+            True,
+        )
+        assert self.stockfish.process.returncode == 0
+
+    def test_bench_128_threads_3_bench_tmp_epd_depth(self):
+        self.stockfish = Stockfish(
+            f"bench 128 {get_threads()} 3 {os.path.join(PATH,'bench_tmp.epd')} depth".split(
+                " "
+            ),
+            True,
+        )
+        assert self.stockfish.process.returncode == 0
+
+    def test_d(self):
+        self.stockfish = Stockfish("d".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_compiler(self):
+        self.stockfish = Stockfish("compiler".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_license(self):
+        self.stockfish = Stockfish("license".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_uci(self):
+        self.stockfish = Stockfish("uci".split(" "), True)
+        assert self.stockfish.process.returncode == 0
+
+    def test_export_net_verify_nnue(self):
+        current_path = os.path.abspath(os.getcwd())
+        self.stockfish = Stockfish(
+            f"export_net {os.path.join(current_path , 'verify.nnue')}".split(" "), True
+        )
+        assert self.stockfish.process.returncode == 0
+
+    # verify the generated net equals the base net
+
+    def test_network_equals_base(self):
+        self.stockfish = Stockfish(
+            ["uci"],
+            True,
+        )
+
+        output = self.stockfish.process.stdout
+
+        # find line
+        for line in output.split("\n"):
+            if "option name EvalFile type string default" in line:
+                network = line.split(" ")[-1]
+                break
+
+        # find network file in src dir
+        network = os.path.join(PATH.parent.resolve(), "src", network)
+
+        if not os.path.exists(network):
+            print(
+                f"Network file {network} not found, please download the network file over the make command."
+            )
+            assert False
+
+        diff = subprocess.run(["diff", network, f"verify.nnue"])
+
+        assert diff.returncode == 0
+
+
+class TestInteractive(metaclass=OrderedClassMembers):
+    def beforeAll(self):
+        self.stockfish = Stockfish()
+
+    def afterAll(self):
+        self.stockfish.quit()
+        assert self.stockfish.close() == 0
+
+    def afterEach(self):
+        assert postfix_check(self.stockfish.get_output()) == True
+        self.stockfish.clear_output()
+
+    def test_startup_output(self):
+        self.stockfish.starts_with("Stockfish")
+
+    def test_uci_command(self):
+        self.stockfish.send_command("uci")
+        self.stockfish.equals("uciok")
+
+    def test_set_threads_option(self):
+        self.stockfish.send_command(f"setoption name Threads value {get_threads()}")
+
+    def test_ucinewgame_and_startpos_nodes_1000(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position startpos")
+        self.stockfish.send_command("go nodes 1000")
+        self.stockfish.starts_with("bestmove")
+
+    def test_ucinewgame_and_startpos_moves(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position startpos moves e2e4 e7e6")
+        self.stockfish.send_command("go nodes 1000")
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_1(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1")
+        self.stockfish.send_command("go nodes 1000")
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_2_flip(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1")
+        self.stockfish.send_command("flip")
+        self.stockfish.send_command("go nodes 1000")
+        self.stockfish.starts_with("bestmove")
+
+    def test_depth_5_with_callback(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position startpos")
+        self.stockfish.send_command("go depth 5")
+
+        def callback(output):
+            regex = "info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv"
+            if output.startswith("info depth") and not re.match(regex, output):
+                assert False
+            if output.startswith("bestmove"):
+                return True
+            return False
+
+        self.stockfish.check_output(callback)
+
+    def test_ucinewgame_and_go_depth_9(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("setoption name UCI_ShowWDL value true")
+        self.stockfish.send_command("position startpos")
+        self.stockfish.send_command("go depth 9")
+
+        depth = 1
+
+        def callback(output):
+            nonlocal depth
+
+            regex = f"info depth {depth} seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv"
+
+            if output.startswith("info depth"):
+                if not re.match(regex, output):
+                    assert False
+                depth += 1
+
+            if output.startswith("bestmove"):
+                assert depth == 10
+                return True
+
+            return False
+
+        self.stockfish.check_output(callback)
+
+    def test_clear_hash(self):
+        self.stockfish.send_command("setoption name Clear Hash")
+
+    def test_fen_position_mate_1(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6"
+        )
+        self.stockfish.send_command("go depth 18")
+
+        self.stockfish.expect("* score mate 1 * pv d5e6")
+        self.stockfish.equals("bestmove d5e6")
+
+    def test_fen_position_mate_minus_1(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -"
+        )
+        self.stockfish.send_command("go depth 18")
+        self.stockfish.expect("* score mate -1 *")
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_fixed_node(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 5K2/8/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1"
+        )
+        self.stockfish.send_command("go nodes 500000")
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_with_mate_go_depth(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
+        )
+        self.stockfish.send_command("go depth 18 searchmoves c6d7")
+        self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5")
+
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_with_mate_go_mate(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
+        )
+        self.stockfish.send_command("go mate 2 searchmoves c6d7")
+        self.stockfish.expect("* score mate 2 * pv c6d7 *")
+
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_with_mate_go_nodes(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
+        )
+        self.stockfish.send_command("go nodes 500000 searchmoves c6d7")
+        self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5")
+
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_depth_27(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - -"
+        )
+        self.stockfish.send_command("go depth 27")
+        self.stockfish.contains("score mate -2")
+
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_with_mate_go_depth_and_promotion(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q"
+        )
+        self.stockfish.send_command("go depth 18")
+        self.stockfish.expect("* score mate 1 * pv f7f5")
+        self.stockfish.starts_with("bestmove f7f5")
+
+    def test_fen_position_with_mate_go_depth_and_searchmoves(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -"
+        )
+        self.stockfish.send_command("go depth 18 searchmoves c6d7")
+        self.stockfish.expect("* score mate 2 * pv c6d7 * f7f5")
+
+        self.stockfish.starts_with("bestmove c6d7")
+
+    def test_fen_position_with_moves_with_mate_go_depth_and_searchmoves(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command(
+            "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7"
+        )
+        self.stockfish.send_command("go depth 18 searchmoves e3e2")
+        self.stockfish.expect("* score mate -1 * pv e3e2 f7f5")
+        self.stockfish.starts_with("bestmove e3e2")
+
+    def test_verify_nnue_network(self):
+        current_path = os.path.abspath(os.getcwd())
+        Stockfish(
+            f"export_net {os.path.join(current_path , 'verify.nnue')}".split(" "), True
+        )
+
+        self.stockfish.send_command("setoption name EvalFile value verify.nnue")
+        self.stockfish.send_command("position startpos")
+        self.stockfish.send_command("go depth 5")
+        self.stockfish.starts_with("bestmove")
+
+    def test_multipv_setting(self):
+        self.stockfish.send_command("setoption name MultiPV value 4")
+        self.stockfish.send_command("position startpos")
+        self.stockfish.send_command("go depth 5")
+        self.stockfish.starts_with("bestmove")
+
+    def test_fen_position_with_skill_level(self):
+        self.stockfish.send_command("setoption name Skill Level value 10")
+        self.stockfish.send_command("position startpos")
+        self.stockfish.send_command("go depth 5")
+        self.stockfish.starts_with("bestmove")
+
+        self.stockfish.send_command("setoption name Skill Level value 20")
+
+
+class TestSyzygy(metaclass=OrderedClassMembers):
+    def beforeAll(self):
+        self.stockfish = Stockfish()
+
+    def afterAll(self):
+        self.stockfish.quit()
+        assert self.stockfish.close() == 0
+
+    def afterEach(self):
+        assert postfix_check(self.stockfish.get_output()) == True
+        self.stockfish.clear_output()
+
+    def test_syzygy_setup(self):
+        self.stockfish.starts_with("Stockfish")
+        self.stockfish.send_command("uci")
+        self.stockfish.send_command(
+            f"setoption name SyzygyPath value {os.path.join(PATH, 'syzygy')}"
+        )
+        self.stockfish.expect(
+            "info string Found 35 WDL and 35 DTZ tablebase files (up to 4-man)."
+        )
+
+    def test_syzygy_bench(self):
+        self.stockfish.send_command("bench 128 1 8 default depth")
+        self.stockfish.expect("Nodes searched  :*")
+
+    def test_syzygy_position(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1")
+        self.stockfish.send_command("go depth 5")
+
+        def check_output(output):
+            if "score cp 20000" in output or "score mate" in output:
+                return True
+
+        self.stockfish.check_output(check_output)
+        self.stockfish.expect("bestmove *")
+
+    def test_syzygy_position_2(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1")
+        self.stockfish.send_command("go depth 5")
+
+        def check_output(output):
+            if "score cp 20000" in output or "score mate" in output:
+                return True
+
+        self.stockfish.check_output(check_output)
+        self.stockfish.expect("bestmove *")
+
+    def test_syzygy_position_3(self):
+        self.stockfish.send_command("ucinewgame")
+        self.stockfish.send_command("position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1")
+        self.stockfish.send_command("go depth 5")
+
+        def check_output(output):
+            if "score cp -20000" in output or "score mate" in output:
+                return True
+
+        self.stockfish.check_output(check_output)
+        self.stockfish.expect("bestmove *")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Run Stockfish with testing options")
+    parser.add_argument("--valgrind", action="store_true", help="Run valgrind testing")
+    parser.add_argument(
+        "--valgrind-thread", action="store_true", help="Run valgrind-thread testing"
+    )
+    parser.add_argument(
+        "--sanitizer-undefined",
+        action="store_true",
+        help="Run sanitizer-undefined testing",
+    )
+    parser.add_argument(
+        "--sanitizer-thread", action="store_true", help="Run sanitizer-thread testing"
+    )
+
+    parser.add_argument(
+        "--none", action="store_true", help="Run without any testing options"
+    )
+    parser.add_argument("stockfish_path", type=str, help="Path to Stockfish binary")
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    EPD.create_bench_epd()
+    TSAN.set_tsan_option()
+    Syzygy.download_syzygy()
+
+    framework = MiniTestFramework()
+
+    # Each test suite will be ran inside a temporary directory
+    framework.run([TestCLI, TestInteractive, TestSyzygy])
+
+    EPD.delete_bench_epd()
+    TSAN.unset_tsan_option()
+
+    if framework.has_failed():
+        sys.exit(1)
+
+    sys.exit(0)
diff --git a/tests/instrumented.sh b/tests/instrumented.sh
deleted file mode 100755
index 5fc6ca9a974..00000000000
--- a/tests/instrumented.sh
+++ /dev/null
@@ -1,301 +0,0 @@
-#!/bin/bash
-# check for errors under Valgrind or sanitizers.
-
-error()
-{
-  echo "instrumented testing failed on line $1"
-  exit 1
-}
-trap 'error ${LINENO}' ERR
-
-# define suitable post and prefixes for testing options
-case $1 in
-  --valgrind)
-    echo "valgrind testing started"
-    prefix=''
-    exeprefix='valgrind --error-exitcode=42 --errors-for-leak-kinds=all --leak-check=full'
-    postfix=''
-    threads="1"
-  ;;
-  --valgrind-thread)
-    echo "valgrind-thread testing started"
-    prefix=''
-    exeprefix='valgrind --fair-sched=try --error-exitcode=42'
-    postfix=''
-    threads="2"
-  ;;
-  --sanitizer-undefined)
-    echo "sanitizer-undefined testing started"
-    prefix='!'
-    exeprefix=''
-    postfix='2>&1 | grep -A50 "runtime error:"'
-    threads="1"
-  ;;
-  --sanitizer-thread)
-    echo "sanitizer-thread testing started"
-    prefix='!'
-    exeprefix=''
-    postfix='2>&1 | grep -A50 "WARNING: ThreadSanitizer:"'
-    threads="2"
-
-cat << EOF > tsan.supp
-race:Stockfish::TTEntry::read
-race:Stockfish::TTEntry::save
-
-race:Stockfish::TranspositionTable::probe
-race:Stockfish::TranspositionTable::hashfull
-
-EOF
-
-    export TSAN_OPTIONS="suppressions=./tsan.supp"
-
-  ;;
-  *)
-    echo "unknown testing started"
-    prefix=''
-    exeprefix=''
-    postfix=''
-    threads="1"
-  ;;
-esac
-
-cat << EOF > bench_tmp.epd
-Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26
-rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3
-3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28
-r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13
-EOF
-
-# simple command line testing
-for args in "eval" \
-            "go nodes 1000" \
-            "go depth 10" \
-            "go perft 4" \
-            "go movetime 1000" \
-            "go wtime 8000 btime 8000 winc 500 binc 500" \
-            "go wtime 1000 btime 1000 winc 0 binc 0" \
-            "go wtime 1000 btime 1000 winc 0 binc 0" \
-            "go wtime 1000 btime 1000 winc 0 binc 0 movestogo 5" \
-            "go movetime 200" \
-            "go nodes 20000 searchmoves e2e4 d2d4" \
-            "bench 128 $threads 8 default depth" \
-            "bench 128 $threads 3 bench_tmp.epd depth" \
-            "export_net verify.nnue" \
-            "d" \
-            "compiler" \
-            "license" \
-            "uci"
-do
-
-   echo "$prefix $exeprefix ./stockfish $args $postfix"
-   eval "$prefix $exeprefix ./stockfish $args $postfix"
-
-done
-
-# verify the generated net equals the base net
-network=`./stockfish uci | grep 'option name EvalFile type string default' | awk '{print $NF}'`
-echo "Comparing $network to the written verify.nnue"
-diff $network verify.nnue
-
-# more general testing, following an uci protocol exchange
-cat << EOF > game.exp
- set timeout 240
- # to correctly catch eof we need the following line
- # expect_before timeout { exit 2 } eof { exit 3 }
- expect_before timeout { exit 2 }
-
- spawn $exeprefix ./stockfish
- expect "Stockfish"
-
- send "uci\n"
- expect "uciok"
-
- # send "setoption name Debug Log File value debug.log\n"
- send "setoption name Threads value $threads\n"
-
- send "ucinewgame\n"
- send "position startpos\n"
- send "go nodes 1000\n"
- expect "bestmove"
-
- send "ucinewgame\n"
- send "position startpos moves e2e4 e7e6\n"
- send "go nodes 1000\n"
- expect "bestmove"
-
- send "ucinewgame\n"
- send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
- send "go depth 10\n"
- expect "bestmove"
-
- send "ucinewgame\n"
- send "position fen 5rk1/1K4p1/8/8/3B4/8/8/8 b - - 0 1\n"
- send "flip\n"
- send "go depth 10\n"
- expect "bestmove"
-
- send "ucinewgame\n"
- send "position startpos\n"
- send "go depth 5\n"
- expect -re {info depth \d+ seldepth \d+ multipv \d+ score cp \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect "bestmove"
-
- send "ucinewgame\n"
- send "setoption name UCI_ShowWDL value true\n"
- send "position startpos\n"
- send "go depth 9\n"
- expect -re {info depth 1 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 2 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 3 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 4 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 5 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 6 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 7 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 8 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect -re {info depth 9 seldepth \d+ multipv \d+ score cp \d+ wdl \d+ \d+ \d+ nodes \d+ nps \d+ hashfull \d+ tbhits \d+ time \d+ pv}
- expect "bestmove"
-
- send "setoption name Clear Hash\n"
-
- send "ucinewgame\n"
- send "position fen 5K2/8/2qk4/2nPp3/3r4/6B1/B7/3R4 w - e6\n"
- send "go depth 18\n"
- expect "score mate 1"
- expect "pv d5e6"
- expect "bestmove d5e6"
-
- send "ucinewgame\n"
- send "position fen 2brrb2/8/p7/Q7/1p1kpPp1/1P1pN1K1/3P4/8 b - -\n"
- send "go depth 18\n"
- expect "score mate -1"
- expect "bestmove"
-
- send "ucinewgame\n"
- send "position fen 7K/P1p1p1p1/2P1P1Pk/6pP/3p2P1/1P6/3P4/8 w - - 0 1\n"
- send "go nodes 500000\n"
- expect "bestmove"
-
- send "ucinewgame\n"
- send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
- send "go depth 18 searchmoves c6d7\n"
- expect "score mate 2 * pv c6d7 * f7f5"
- expect "bestmove c6d7"
-
- send "ucinewgame\n"
- send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
- send "go mate 2 searchmoves c6d7\n"
- expect "score mate 2 * pv c6d7"
- expect "bestmove c6d7"
-
- send "ucinewgame\n"
- send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
- send "go nodes 500000 searchmoves c6d7\n"
- expect "score mate 2 * pv c6d7 * f7f5"
- expect "bestmove c6d7"
-
- send "ucinewgame\n"
- send "position fen 1NR2B2/5p2/5p2/1p1kpp2/1P2rp2/2P1pB2/2P1P1K1/8 b - - \n"
- send "go depth 27\n"
- expect "score mate -2"
- expect "pv d5e6 c8d8"
- expect "bestmove d5e6"
-
- send "ucinewgame\n"
- send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7 f2f1q\n"
- send "go depth 18\n"
- expect "score mate 1 * pv f7f5"
- expect "bestmove f7f5"
- 
- send "ucinewgame\n"
- send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - -\n"
- send "go depth 18 searchmoves c6d7\n"
- expect "score mate 2 * pv c6d7 * f7f5"
- expect "bestmove c6d7"
- 
- send "ucinewgame\n"
- send "position fen 8/5R2/2K1P3/4k3/8/b1PPpp1B/5p2/8 w - - moves c6d7\n"
- send "go depth 18 searchmoves e3e2\n"
- expect "score mate -1 * pv e3e2 f7f5"
- expect "bestmove e3e2"
-
- send "setoption name EvalFile value verify.nnue\n"
- send "position startpos\n"
- send "go depth 5\n"
- expect "bestmove"
-
- send "setoption name MultiPV value 4\n"
- send "position startpos\n"
- send "go depth 5\n"
- expect "bestmove"
-
- send "setoption name Skill Level value 10\n"
- send "position startpos\n"
- send "go depth 5\n"
- expect "bestmove"
- send "setoption name Skill Level value 20\n"
-
- send "quit\n"
- expect eof
-
- # return error code of the spawned program, useful for Valgrind
- lassign [wait] pid spawnid os_error_flag value
- exit \$value
-EOF
-
-#download TB as needed
-if [ ! -d ../tests/syzygy ]; then
-   curl -sL https://api.github.com/repos/niklasf/python-chess/tarball/9b9aa13f9f36d08aadfabff872882f4ab1494e95 | tar -xzf -
-   mv niklasf-python-chess-9b9aa13 ../tests/syzygy
-fi
-
-cat << EOF > syzygy.exp
- set timeout 240
- # to correctly catch eof we need the following line
- # expect_before timeout { exit 2 } eof { exit 3 }
- expect_before timeout { exit 2 }
- spawn $exeprefix ./stockfish
- expect "Stockfish"
- send "uci\n"
- send "setoption name SyzygyPath value ../tests/syzygy/\n"
- expect "info string Found 35 WDL and 35 DTZ tablebase files (up to 4-man)."
- send "bench 128 1 8 default depth\n"
- expect "Nodes searched  :"
- send "ucinewgame\n"
- send "position fen 4k3/PP6/8/8/8/8/8/4K3 w - - 0 1\n"
- send "go depth 5\n"
- expect -re {score cp 20000|score mate}
- expect "bestmove"
- send "ucinewgame\n"
- send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 w - - 0 1\n"
- send "go depth 5\n"
- expect -re {score cp 20000|score mate}
- expect "bestmove"
- send "ucinewgame\n"
- send "position fen 8/1P6/2B5/8/4K3/8/6k1/8 b - - 0 1\n"
- send "go depth 5\n"
- expect -re {score cp -20000|score mate}
- expect "bestmove"
- send "quit\n"
- expect eof
-
- # return error code of the spawned program, useful for Valgrind
- lassign [wait] pid spawnid os_error_flag value
- exit \$value
-EOF
-
-for exp in game.exp syzygy.exp
-do
-
-  echo "======== $exp =============="
-  cat $exp
-  echo "============================"
-  echo "$prefix expect $exp $postfix"
-  eval "$prefix expect $exp $postfix"
-
-  rm $exp
-
-done
-
-rm -f tsan.supp bench_tmp.epd
-
-echo "instrumented testing OK"
diff --git a/tests/testing.py b/tests/testing.py
new file mode 100644
index 00000000000..3034ac737f0
--- /dev/null
+++ b/tests/testing.py
@@ -0,0 +1,372 @@
+import subprocess
+from typing import List, Callable, Any
+import os
+import collections
+import time
+import sys
+import traceback
+import fnmatch
+from functools import wraps
+from contextlib import redirect_stdout
+import io
+import tarfile
+import urllib.request
+import pathlib
+import concurrent.futures
+import tempfile
+
+CYAN_COLOR = "\033[36m"
+GRAY_COLOR = "\033[2m"
+RED_COLOR = "\033[31m"
+GREEN_COLOR = "\033[32m"
+RESET_COLOR = "\033[0m"
+WHITE_BOLD = "\033[1m"
+
+MAX_TIMEOUT = 60 * 5
+
+PATH = pathlib.Path(__file__).parent.resolve()
+
+
+class Valgrind:
+    @staticmethod
+    def get_valgrind_command():
+        return [
+            "valgrind",
+            "--error-exitcode=42",
+            "--errors-for-leak-kinds=all",
+            "--leak-check=full",
+        ]
+
+    @staticmethod
+    def get_valgrind_thread_command():
+        return ["valgrind", "--error-exitcode=42", "--fair-sched=try"]
+
+
+class TSAN:
+    @staticmethod
+    def set_tsan_option():
+        with open(f"tsan.supp", "w") as f:
+            f.write(
+                """
+race:Stockfish::TTEntry::read
+race:Stockfish::TTEntry::save
+race:Stockfish::TranspositionTable::probe
+race:Stockfish::TranspositionTable::hashfull
+"""
+            )
+
+        os.environ["TSAN_OPTIONS"] = "suppressions=./tsan.supp"
+
+    @staticmethod
+    def unset_tsan_option():
+        os.environ.pop("TSAN_OPTIONS", None)
+        os.remove(f"tsan.supp")
+
+
+class EPD:
+    @staticmethod
+    def create_bench_epd():
+        with open(f"{os.path.join(PATH,'bench_tmp.epd')}", "w") as f:
+            f.write(
+                """
+Rn6/1rbq1bk1/2p2n1p/2Bp1p2/3Pp1pP/1N2P1P1/2Q1NPB1/6K1 w - - 2 26
+rnbqkb1r/ppp1pp2/5n1p/3p2p1/P2PP3/5P2/1PP3PP/RNBQKBNR w KQkq - 0 3
+3qnrk1/4bp1p/1p2p1pP/p2bN3/1P1P1B2/P2BQ3/5PP1/4R1K1 w - - 9 28
+r4rk1/1b2ppbp/pq4pn/2pp1PB1/1p2P3/1P1P1NN1/1PP3PP/R2Q1RK1 w - - 0 13
+"""
+            )
+
+    @staticmethod
+    def delete_bench_epd():
+        os.remove(f"{os.path.join(PATH,'bench_tmp.epd')}")
+
+
+class Syzygy:
+    @staticmethod
+    def get_syzygy_path():
+        return os.path.abspath("syzygy")
+
+    @staticmethod
+    def download_syzygy():
+        if not os.path.isdir(os.path.join(PATH, "syzygy")):
+            url = "https://api.github.com/repos/niklasf/python-chess/tarball/9b9aa13f9f36d08aadfabff872882f4ab1494e95"
+            tarball_path = "/tmp/python-chess.tar.gz"
+
+            urllib.request.urlretrieve(url, tarball_path)
+
+            with tarfile.open(tarball_path, "r:gz") as tar:
+                tar.extractall("/tmp")
+
+            os.rename("/tmp/niklasf-python-chess-9b9aa13", os.path.join(PATH, "syzygy"))
+
+
+class OrderedClassMembers(type):
+    @classmethod
+    def __prepare__(self, name, bases):
+        return collections.OrderedDict()
+
+    def __new__(self, name, bases, classdict):
+        classdict["__ordered__"] = [
+            key for key in classdict.keys() if key not in ("__module__", "__qualname__")
+        ]
+        return type.__new__(self, name, bases, classdict)
+
+
+class TimeoutException(Exception):
+    def __init__(self, message: str, timeout: int):
+        self.message = message
+        self.timeout = timeout
+
+
+def timeout_decorator(timeout: float):
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                future = executor.submit(func, *args, **kwargs)
+                try:
+                    result = future.result(timeout=timeout)
+                except concurrent.futures.TimeoutError:
+                    raise TimeoutException(
+                        f"Function {func.__name__} timed out after {timeout} seconds",
+                        timeout,
+                    )
+            return result
+
+        return wrapper
+
+    return decorator
+
+
+class MiniTestFramework:
+    def __init__(self):
+        self.passed_test_suites = 0
+        self.failed_test_suites = 0
+        self.passed_tests = 0
+        self.failed_tests = 0
+
+    def has_failed(self) -> bool:
+        return self.failed_test_suites > 0
+
+    def run(self, classes: List[type]) -> bool:
+        self.start_time = time.time()
+
+        for test_class in classes:
+            with tempfile.TemporaryDirectory() as tmpdirname:
+                original_cwd = os.getcwd()
+                os.chdir(tmpdirname)
+
+                try:
+                    if self.__run(test_class):
+                        self.failed_test_suites += 1
+                    else:
+                        self.passed_test_suites += 1
+                finally:
+                    os.chdir(original_cwd)
+
+        self.__print_summary(round(time.time() - self.start_time, 2))
+        return self.has_failed()
+
+    def __run(self, test_class) -> bool:
+        test_instance = test_class()
+        test_name = test_instance.__class__.__name__
+        test_methods = [m for m in test_instance.__ordered__ if m.startswith("test_")]
+
+        print(f"\nTest Suite: {test_name}")
+
+        if hasattr(test_instance, "beforeAll"):
+            test_instance.beforeAll()
+
+        fails = 0
+
+        for method in test_methods:
+            fails += self.__run_test_method(test_instance, method)
+
+        if hasattr(test_instance, "afterAll"):
+            test_instance.afterAll()
+
+        self.failed_tests += fails
+
+        return fails > 0
+
+    def __run_test_method(self, test_instance, method: str) -> int:
+        print(f"    Running {method}... \r", end="", flush=True)
+
+        buffer = io.StringIO()
+        fails = 0
+
+        try:
+            t0 = time.time()
+
+            with redirect_stdout(buffer):
+                if hasattr(test_instance, "beforeEach"):
+                    test_instance.beforeEach()
+
+                getattr(test_instance, method)()
+
+                if hasattr(test_instance, "afterEach"):
+                    test_instance.afterEach()
+
+            duration = time.time() - t0
+
+            self.print_success(f" {method} ({duration * 1000:.2f}ms)")
+            self.passed_tests += 1
+        except Exception as e:
+            if isinstance(e, TimeoutException):
+                self.print_failure(
+                    f" {method} (hit execution limit of {e.timeout} seconds)"
+                )
+
+            if isinstance(e, AssertionError):
+                self.__handle_assertion_error(t0, method)
+
+            fails += 1
+        finally:
+            self.__print_buffer_output(buffer)
+
+        return fails
+
+    def __handle_assertion_error(self, start_time, method: str):
+        duration = time.time() - start_time
+        self.print_failure(f" {method} ({duration * 1000:.2f}ms)")
+        traceback_output = "".join(traceback.format_tb(sys.exc_info()[2]))
+
+        colored_traceback = "\n".join(
+            f"  {CYAN_COLOR}{line}{RESET_COLOR}"
+            for line in traceback_output.splitlines()
+        )
+
+        print(colored_traceback)
+
+    def __print_buffer_output(self, buffer: io.StringIO):
+        output = buffer.getvalue()
+        if output:
+            indented_output = "\n".join(f"    {line}" for line in output.splitlines())
+            print(f"    {RED_COLOR}⎯⎯⎯⎯⎯OUTPUT⎯⎯⎯⎯⎯{RESET_COLOR}")
+            print(f"{GRAY_COLOR}{indented_output}{RESET_COLOR}")
+            print(f"    {RED_COLOR}⎯⎯⎯⎯⎯OUTPUT⎯⎯⎯⎯⎯{RESET_COLOR}")
+
+    def __print_summary(self, duration: float):
+        print(f"\n{WHITE_BOLD}Test Summary{RESET_COLOR}\n")
+        print(
+            f"    Test Suites: {GREEN_COLOR}{self.passed_test_suites} passed{RESET_COLOR}, {RED_COLOR}{self.failed_test_suites} failed{RESET_COLOR}, {self.passed_test_suites + self.failed_test_suites} total"
+        )
+        print(
+            f"    Tests:       {GREEN_COLOR}{self.passed_tests} passed{RESET_COLOR}, {RED_COLOR}{self.failed_tests} failed{RESET_COLOR}, {self.passed_tests + self.failed_tests} total"
+        )
+        print(f"    Time:        {duration}s\n")
+
+    def print_failure(self, add: str):
+        print(f"    {RED_COLOR}✗{RESET_COLOR}{add}", flush=True)
+
+    def print_success(self, add: str):
+        print(f"    {GREEN_COLOR}✓{RESET_COLOR}{add}", flush=True)
+
+
+class Stockfish:
+    def __init__(
+        self,
+        prefix: List[str],
+        path: str,
+        args: List[str] = [],
+        cli: bool = False,
+    ):
+        self.path = path
+        self.process = None
+        self.args = args
+        self.cli = cli
+        self.prefix = prefix
+        self.output = []
+
+        self.start()
+
+    def start(self):
+        if self.cli:
+            self.process = subprocess.run(
+                self.prefix + [self.path] + self.args,
+                capture_output=True,
+                text=True,
+            )
+
+            self.process.stdout
+
+            return
+
+        self.process = subprocess.Popen(
+            self.prefix + [self.path] + self.args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            bufsize=1,
+        )
+
+    def setoption(self, name: str, value: str):
+        self.send_command(f"setoption name {name} value {value}")
+
+    def send_command(self, command: str):
+        if not self.process:
+            raise RuntimeError("Stockfish process is not started")
+
+        self.process.stdin.write(command + "\n")
+        self.process.stdin.flush()
+
+    @timeout_decorator(MAX_TIMEOUT)
+    def equals(self, expected_output: str):
+        for line in self.readline():
+            if line == expected_output:
+                return
+
+    @timeout_decorator(MAX_TIMEOUT)
+    def expect(self, expected_output: str):
+        for line in self.readline():
+            if fnmatch.fnmatch(line, expected_output):
+                return
+
+    @timeout_decorator(MAX_TIMEOUT)
+    def contains(self, expected_output: str):
+        for line in self.readline():
+            if expected_output in line:
+                return
+
+    @timeout_decorator(MAX_TIMEOUT)
+    def starts_with(self, expected_output: str):
+        for line in self.readline():
+            if line.startswith(expected_output):
+                return
+
+    @timeout_decorator(MAX_TIMEOUT)
+    def check_output(self, callback):
+        if not callback:
+            raise ValueError("Callback function is required")
+
+        for line in self.readline():
+            if callback(line) == True:
+                return
+
+    def readline(self):
+        if not self.process:
+            raise RuntimeError("Stockfish process is not started")
+
+        while True:
+            line = self.process.stdout.readline().strip()
+            self.output.append(line)
+
+            yield line
+
+    def clear_output(self):
+        self.output = []
+
+    def get_output(self) -> List[str]:
+        return self.output
+
+    def quit(self):
+        self.send_command("quit")
+
+    def close(self):
+        if self.process:
+            self.process.stdin.close()
+            self.process.stdout.close()
+            return self.process.wait()
+
+        return 0