Fix NPS measurement for TC scaling

worker/games.py

@@ -322,6 +322,30 @@ def establish_validated_net(remote, testing_dir, net):
                     )
                 )
                 time.sleep(waitTime)
+
+def run_single_bench(engine, queue):
+    bench_sig = None
+    bench_nps = None
+
+    try:
+        p = subprocess.Popen(
+            [engine, "bench"],
+            stderr=subprocess.PIPE,
+            stdout=subprocess.DEVNULL,
+            universal_newlines=True,
+            bufsize=1,
+            close_fds=not IS_WINDOWS,
+        )
+
+        for line in iter(p.stderr.readline, ""):
+            if "Nodes searched" in line:
+                bench_sig = line.split(": ")[1].strip()
+            if "Nodes/second" in line:
+                bench_nps = float(line.split(": ")[1].strip())
+
+        queue.put((bench_sig, bench_nps))
+    except:
+        queue.put((None, None))
 
 
 def verify_signature(engine, signature, active_cores):
@@ -344,74 +368,39 @@ def verify_signature(engine, signature, active_cores):
             )
         )
 
-    with ExitStack() as stack:
-        if active_cores > 1:
-            busy_process = stack.enter_context(
-                subprocess.Popen(
-                    [engine],
-                    stdin=subprocess.PIPE,
-                    stdout=subprocess.DEVNULL,
-                    universal_newlines=True,
-                    bufsize=1,
-                    close_fds=not IS_WINDOWS,
-                )
-            )
-            busy_process.stdin.write(
-                "setoption name Threads value {}\n".format(active_cores - 1)
-            )
-            busy_process.stdin.write("go infinite\n")
-            busy_process.stdin.flush()
-            time.sleep(1)  # wait CPU loading
-
-        bench_sig = None
-        bench_nps = None
-        print("Verifying signature of {} ...".format(os.path.basename(engine)))
-        p = stack.enter_context(
-            subprocess.Popen(
-                [engine, "bench"],
-                stderr=subprocess.PIPE,
-                stdout=subprocess.DEVNULL,
-                universal_newlines=True,
-                bufsize=1,
-                close_fds=not IS_WINDOWS,
-            )
-        )
-        for line in iter(p.stderr.readline, ""):
-            if "Nodes searched" in line:
-                bench_sig = line.split(": ")[1].strip()
-            if "Nodes/second" in line:
-                bench_nps = float(line.split(": ")[1].strip())
+    queue = multiprocessing.Queue()
 
-        if active_cores > 1:
-            busy_process.communicate("quit\n")
+    processes = [
+        multiprocessing.Process(
+            target=run_single_bench,
+            args=(engine, queue),
+        ) for _ in range(active_cores)
+    ]
 
-    if p.returncode != 0:
-        if p.returncode == 1:  # EXIT_FAILURE
+    for p in processes:
+        p.start()
+
+    results = [queue.get() for _ in range(active_cores)]
+    bench_nps = 0.0
+
+    for sig, nps in results:
+
+        if sig is None or bench_nps is None:
             raise RunException(
-                "Bench of {} exited with EXIT_FAILURE".format(os.path.basename(engine))
+                "Unable to parse bench output of {}".format(os.path.basename(engine))
             )
-        else:  # Signal? It could be user generated so be careful.
-            raise WorkerException(
-                "Bench of {} exited with error code {}".format(
-                    os.path.basename(engine), format_return_code(p.returncode)
-                )
-            )
-
-    # Now we know that bench finished without error we check that its
-    # output is correct.
 
-    if bench_sig is None or bench_nps is None:
-        raise RunException(
-            "Unable to parse bench output of {}".format(os.path.basename(engine))
-        )
+        if int(sig) != int(signature):
+            message = "Wrong bench in {}, user expected: {} but worker got: {}".format(
+                os.path.basename(engine),
+                signature,
+                sig,
+            )
+            raise RunException(message)
+
+        bench_nps += nps
 
-    if int(bench_sig) != int(signature):
-        message = "Wrong bench in {}, user expected: {} but worker got: {}".format(
-            os.path.basename(engine),
-            signature,
-            bench_sig,
-        )
-        raise RunException(message)
+    bench_nps /= active_cores
 
     return bench_nps, cpu_features
 

worker/sri.txt

@@ -1 +1 @@
-{"__version": 239, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "+ubGHk3rIV0ILVhg1dxpsOkRF8GUaO5otPVnAyw8kKKq9Rqzksv02xj6wjYpSTmA", "games.py": "6vKH51UtL56oNvA539hLXRzgE1ADXy3QZNJohoK94RntM72+iMancSJZHaNjEb5+"}
+{"__version": 240, "updater.py": "Mg+pWOgGA0gSo2TuXuuLCWLzwGwH91rsW1W3ixg3jYauHQpRMtNdGnCfuD1GqOhV", "worker.py": "A8axNSsOlKlQLTjI5Tn6sjpfEfRxVEawLOMEi3JbHZkd0wJ+Lw3TfI+11gHviTD/", "games.py": "okZWdH4eUKbLnvMf2yfEqB8GAw/HxHb5uIe1pS4svGppN5mtvfeWwTNg6XFPUOuo"}

worker/worker.py

@@ -62,7 +62,7 @@
 MIN_CLANG_MAJOR = 8
 MIN_CLANG_MINOR = 0
 
-WORKER_VERSION = 239
+WORKER_VERSION = 240
 FILE_LIST = ["updater.py", "worker.py", "games.py"]
 HTTP_TIMEOUT = 30.0
 INITIAL_RETRY_TIME = 15.0