test: always kill warnet server

Currently, if a test fails warnet server might not be properly killed
and so remain running.

This is annoying for iteration; if the test fails we want everything
gone ready to run again.

Also only try to bring the network down if we first brought it up.

test/test_base.py

@@ -1,4 +1,6 @@
 import atexit
+import os
+import signal
 from pathlib import Path
 from subprocess import Popen, run, PIPE
 from tempfile import mkdtemp
@@ -17,47 +19,66 @@ def __init__(self):
         self.network_name = self.tmpdir.name
         self.logfile = None
         self.server = None
+        self.network = None
 
         atexit.register(self.cleanup)
 
         print(f"\nWarnet test base started")
 
-
-    def cleanup(self, signum = None, frame = None):
-        if self.server is None:
-            return
-
+    def _cleanup_docker(self):
         try:
-            print("\nStopping network")
-            self.warcli("network down")
-
-            self.wait_for_all_tanks_status(target="none", timeout=60, interval=1)
-
-            print("\nStopping server")
-            self.warcli("stop", False)
-        except Exception as e:
-            # Remove the temporary docker network when we quit.
-            # If the warnet server exited prematurely then docker-compose down
-            # likely did not succeed or was never executed.
-            print(f"Error stopping server: {e}")
             print("Attempting to cleanup docker network")
             wn = Warnet.from_network(self.network_name)
             wn.docker_compose_down()
+        except FileNotFoundError:
+            pass
+        except Exception:
+            pass
+
+    def _cleanup_network(self):
+        if self.network is not None:
+            try:
+                print("\nStopping network")
+                self.warcli("network down")
+
+                self.wait_for_all_tanks_status(target="none", timeout=60, interval=1)
+
+                print("\nStopping server")
+                self.warcli("stop", False)
+            except Exception as e:
+                # Remove the temporary docker network when we quit.
+                # If the warnet server exited prematurely then docker-compose down
+                # likely did not succeed or was never executed.
+                print(f"Error stopping server: {e}")
+                self._cleanup_docker()
+
+    def _cleanup_server(self):
+        # kill the server process by its PID if it still survives
+        if self.server:
+            print(f"killing server with pid {self.server.pid}")
+            os.killpg(os.getpgid(self.server.pid), signal.SIGTERM)
+
+    def cleanup(self, signum = None, frame = None):
+        if self.server is None:
+            return
 
-        print("\nRemaining server output:")
-        print(self.logfile.read())
-        self.logfile.close()
+        self._cleanup_network()
+        self._cleanup_server()
 
-        self.logfile = None
-        self.server.terminate()
-        self.server = None
+        if self.logfile:
+            print("\nRemaining server output:")
+            print(self.logfile.read())
+            self.logfile.close()
+            self.logfile = None
 
 
     # Execute a warcli RPC using command line (always returns string)
     def warcli(self, str, network=True):
         cmd = ["warcli"] + str.split()
         if network:
             cmd += ["--network", self.network_name]
+        if "network" in cmd and "start" in cmd:
+            self.network = 1
         proc = run(
             cmd,
             stdout=PIPE,
@@ -67,6 +88,8 @@ def warcli(self, str, network=True):
 
     # Execute a warnet RPC API call directly (may return dict or list)
     def rpc(self, method, params = []):
+        if "network" in method and "start" in params:
+            self.network = 1
         return rpc_call(method, params)
 
 
@@ -87,7 +110,8 @@ def start_server(self):
         print(f"\nStarting Warnet server, logging to: {self.logfilepath}")
         self.server = Popen(
             f"warnet > {self.logfilepath}",
-            shell=True)
+            shell=True,
+            preexec_fn=os.setsid)  # This sets up a new session and process group
 
         print("\nWaiting for RPC")
         # doesn't require anything docker-related