Switch from cutechess-cli to fast-chess

fixes official-stockfish#2106

This PR switches from cutechess-cli to fast-chess.

cutechess-cli has been serving us well in the past years, however, some issues
have accumulated, namely the difficulty of compiling cutechess-cli, the
observed timeouts at high concurrency and short TC, and e.g. slowness when
indexing larger books.  fast-chess https://github.com/Disservin/fast-chess has
addressed these issues, and has now probably become mature enough to serve as
the game manager for fishtest.

As an example of its ability to deal with short TC and high concurrency:
https://dfts-0.pigazzini.it/tests/view/669249cdbee8253775cede32
with concurrency 25, and TC 1+0.01s no timeouts are observed.

fast-chess is built from sources, with the zip download as well as the binary
cached as needed.  There is fine-grained control over which version of
fast-chess is used, so we can easily upgrade for new features.

In this PR, fast-chess is built in cutechess compatibility to facilitate
integration, and to benefit from the existing fishtest checks. Once validated,
we should be able to switch easily to its native mode, which can output
trinomial and pentanomial results, and we should be able significantly simplify
the worker's book-keeping.

Co-Authored-By: Disservin <[email protected]>
Co-Authored-By: Gahtan Nahdi <[email protected]>

AUTHORS

@@ -14,6 +14,7 @@ Fanael Linithien (Fanael)
 Fauzi Akram Dabat (FauziAkram)
 FieryDragonLord
 Gabe (MrBrain295)
+Gahtan Nahdi (gahtan-syarif)
 Giacomo Lorenzetti (G-Lorenz)
 Gian-Carlo Pascutto (gcp)
 Henri Wiechers (hwiechers)

server/fishtest/api.py

@@ -34,7 +34,7 @@
 according to the route/URL mapping defined in `__init__.py`.
 """
 
-WORKER_VERSION = 241
+WORKER_VERSION = 242
 
 
 @exception_view_config(HTTPException)

server/fishtest/rundb.py

@@ -1158,23 +1158,6 @@ def priority(run):  # lower is better
                 if not have_binary:
                     continue
 
-            # To avoid time losses in the case of large concurrency and short TC,
-            # probably due to cutechess-cli as discussed in issue #822,
-            # assign linux workers to LTC or multi-threaded jobs
-            # and windows workers only to LTC jobs
-            if max_threads >= 29:
-                if "windows" in worker_info["uname"].lower():
-                    tc_too_short = get_tc_ratio(run["args"]["tc"], base="55+0.5") < 1.0
-                else:
-                    tc_too_short = (
-                        get_tc_ratio(
-                            run["args"]["tc"], run["args"]["threads"], "35+0.3"
-                        )
-                        < 1.0
-                    )
-                if tc_too_short:
-                    continue
-
             # Limit the number of cores.
             # Currently this is only done for spsa.
             if "spsa" in run["args"]:

worker/games.py

@@ -67,7 +67,7 @@ def is_64bit():
 
 
 HTTP_TIMEOUT = 30.0
-CUTECHESS_KILL_TIMEOUT = 15.0
+FASTCHESS_KILL_TIMEOUT = 15.0
 UPDATE_RETRY_TIME = 15.0
 
 RAWCONTENT_HOST = "https://raw.githubusercontent.com"
@@ -504,24 +504,6 @@ def unzip(blob, save_dir):
     return file_list
 
 
-def convert_book_move_counters(book_file):
-    # converts files with complete FENs, leaving others (incl. converted ones) unchanged
-    epds = []
-    with open(book_file, "r") as file:
-        for fen in file:
-            fields = fen.split()
-            if len(fields) == 6 and fields[4].isdigit() and fields[5].isdigit():
-                fields[4] = f"hmvc {fields[4]};"
-                fields[5] = f"fmvn {fields[5]};"
-                epds.append(" ".join(fields))
-            else:
-                return
-
-    with open(book_file, "w") as file:
-        for epd in epds:
-            file.write(epd + "\n")
-
-
 def clang_props():
     """Parse the output of clang++ -E - -march=native -### and extract the available clang properties"""
     with subprocess.Popen(
@@ -958,7 +940,7 @@ def results_to_score(results):
     assert abs(s5 - s3) < epsilon
 
 
-def parse_cutechess_output(
+def parse_fastchess_output(
     p, current_state, remote, result, spsa_tuning, games_to_play, batch_size, tc_limit
 ):
     hash_pattern = re.compile(r"(Base|New)-[a-f0-9]+")
@@ -1002,13 +984,13 @@ def shorten_hash(match):
         # Parse line like this:
         # Warning: New-SHA doesn't have option ThreatBySafePawn
         if "Warning:" in line and "doesn't have option" in line:
-            message = r'Cutechess-cli says: "{}"'.format(line)
+            message = r'fast-chess says: "{}"'.format(line)
             raise RunException(message)
 
         # Parse line like this:
         # Warning: Invalid value for option P: -354
         if "Warning:" in line and "Invalid value" in line:
-            message = r'Cutechess-cli says: "{}"'.format(line)
+            message = r'fast-chess says: "{}"'.format(line)
             raise RunException(message)
 
         # Parse line like this:
@@ -1032,7 +1014,7 @@ def shorten_hash(match):
 
             validate_pentanomial(
                 wld, rounds
-            )  # check if cutechess-cli result is compatible with
+            )  # check if fast-chess result is compatible with
             # our own bookkeeping
 
             pentanomial = [
@@ -1123,7 +1105,7 @@ def shorten_hash(match):
     return True
 
 
-def launch_cutechess(
+def launch_fastchess(
     cmd, current_state, remote, result, spsa_tuning, games_to_play, batch_size, tc_limit
 ):
     if spsa_tuning:
@@ -1154,7 +1136,7 @@ def launch_cutechess(
         w_params = []
         b_params = []
 
-    # Run cutechess-cli binary.
+    # Run fast-chess binary.
     # Stochastic rounding and probability for float N.p: (N, 1-p); (N+1, p)
     idx = cmd.index("_spsa_")
     cmd = (
@@ -1179,7 +1161,7 @@ def launch_cutechess(
         + cmd[idx + 1 :]
     )
 
-    #    print(cmd)
+    # print(cmd)
     try:
         with subprocess.Popen(
             cmd,
@@ -1201,7 +1183,7 @@ def launch_cutechess(
             close_fds=not IS_WINDOWS,
         ) as p:
             try:
-                task_alive = parse_cutechess_output(
+                task_alive = parse_fastchess_output(
                     p,
                     current_state,
                     remote,
@@ -1212,28 +1194,28 @@ def launch_cutechess(
                     tc_limit,
                 )
             finally:
-                # We nicely ask cutechess-cli to stop.
+                # We nicely ask fast-chess to stop.
                 try:
                     send_sigint(p)
                 except Exception as e:
                     print("\nException in send_sigint:\n", e, sep="", file=sys.stderr)
                 # now wait...
-                print("\nWaiting for cutechess-cli to finish ... ", end="", flush=True)
+                print("\nWaiting for fast-chess to finish ... ", end="", flush=True)
                 try:
-                    p.wait(timeout=CUTECHESS_KILL_TIMEOUT)
+                    p.wait(timeout=FASTCHESS_KILL_TIMEOUT)
                 except subprocess.TimeoutExpired:
                     print("timeout", flush=True)
                     kill_process(p)
                 else:
                     print("done", flush=True)
     except (OSError, subprocess.SubprocessError) as e:
         print(
-            "Exception starting cutechess:\n",
+            "Exception starting fast-chess:\n",
             e,
             sep="",
             file=sys.stderr,
         )
-        raise WorkerException("Unable to start cutechess. Error: {}".format(str(e)))
+        raise WorkerException("Unable to start fast-chess. Error: {}".format(str(e)))
 
     return task_alive
 
@@ -1249,7 +1231,7 @@ def run_games(
     clear_binaries,
     global_cache,
 ):
-    # This is the main cutechess-cli driver.
+    # This is the main fast-chess driver.
     # It is ok, and even expected, for this function to
     # raise exceptions, implicitly or explicitly, if a
     # task cannot be completed.
@@ -1317,7 +1299,7 @@ def run_games(
     start_game_index = opening_offset + input_total_games
     run_seed = int(hashlib.sha1(run["_id"].encode("utf-8")).hexdigest(), 16) % (2**30)
 
-    # Format options according to cutechess syntax.
+    # Format options according to fastchess syntax.
     def parse_options(s):
         results = []
         chunks = s.split("=")
@@ -1404,11 +1386,6 @@ def parse_options(s):
         blob = download_from_github(zipball)
         unzip(blob, testing_dir)
 
-    # convert .epd containing FENs into .epd containing EPDs with move counters
-    # only needed as long as cutechess-cli is the game manager
-    if book.endswith(".epd"):
-        convert_book_move_counters(testing_dir / book)
-
     # Clean up the old networks (keeping the num_bkps most recent)
     num_bkps = 10
     for old_net in sorted(
@@ -1424,7 +1401,7 @@ def parse_options(s):
                 file=sys.stderr,
             )
 
-    # Add EvalFile* with full path to cutechess options, and download the networks if missing.
+    # Add EvalFile* with full path to fast-chess options, and download the networks if missing.
     for option, net in required_nets(base_engine).items():
         base_options.append("option.{}={}".format(option, net))
         establish_validated_net(remote, testing_dir, net, global_cache)
@@ -1554,15 +1531,17 @@ def make_player(arg):
         if any(substring in book.upper() for substring in ["FRC", "960"]):
             variant = "fischerandom"
 
-        # Run cutechess binary.
-        cutechess = "cutechess-cli" + EXE_SUFFIX
+        # Run fastchess binary.
+        fastchess = "fast-chess" + EXE_SUFFIX
         cmd = (
             [
-                os.path.join(testing_dir, cutechess),
+                os.path.join(testing_dir, fastchess),
                 "-recover",
                 "-repeat",
                 "-games",
-                str(int(games_to_play)),
+                "2",
+                "-rounds",
+                str(int(games_to_play) // 2),
                 "-tournament",
                 "gauntlet",
             ]
@@ -1618,7 +1597,7 @@ def make_player(arg):
             + book_cmd
         )
 
-        task_alive = launch_cutechess(
+        task_alive = launch_fastchess(
             cmd,
             current_state,
             remote,

worker/sri.txt

@@ -1 +1 @@
-{"__version": 241, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "BMuQUpxZAKF0aP6ByTZY1r06MfPoIbdG2xraTrDQQRKgvhzJo6CKmeX2P8vX/QDm", "games.py": "9dFaa914vpqT7q4LLx2LlDdYwK6QFVX3h7+XRt18ATX0lt737rvFeBIiqakkttNC"}
+{"__version": 242, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "fF/2yzobPMjsU42K0O7P0gGn2Ss/gXLXhwAbP0nKkHnGnKQ0WSGEuFiCy/qt2T1E", "games.py": "7RjSD5X3UP0DIgQecSpaRKTVFmf/4POUAHpNskuPVWNkdscE7+2PTQGfqHklIwho"}

worker/tests/test_worker.py

@@ -70,8 +70,12 @@ def test_sri(self):
     def test_toolchain_verification(self):
         self.assertTrue(worker.verify_toolchain())
 
-    def test_setup_cutechess(self):
-        self.assertTrue(worker.setup_cutechess(Path.cwd()))
+    def test_setup_fastchess(self):
+        self.assertTrue(
+            worker.setup_fastchess(
+                Path.cwd(), list(worker.detect_compilers().keys())[0], 4, ""
+            )
+        )
 
 
 if __name__ == "__main__":