Make PyMongo timezone-aware

https://pymongo.readthedocs.io/en/stable/examples/datetimes.html#reading-time

"By default all `datetime.datetime` objects returned by PyMongo
will be naive but reflect UTC (i.e. the time as stored in MongoDB).
By setting the `tz_aware` option on `CodecOptions`,
`datetime.datetime` objects will be timezone-aware
and have a `tzinfo` property that reflects the UTC timezone."

server/fishtest/rundb.py

@@ -15,6 +15,7 @@
 
 import fishtest.stats.stat_util
 from bson.binary import Binary
+from bson.codec_options import CodecOptions
 from bson.objectid import ObjectId
 from fishtest.actiondb import ActionDb
 from fishtest.schemas import RUN_VERSION, nn_schema, runs_schema
@@ -85,7 +86,8 @@ def __init__(self, db_name="fishtest_new"):
         # MongoDB server is assumed to be on the same machine, if not user should
         # use ssh with port forwarding to access the remote host.
         self.conn = MongoClient(os.getenv("FISHTEST_HOST") or "localhost")
-        self.db = self.conn[db_name]
+        codec_options = CodecOptions(tz_aware=True, tzinfo=timezone.utc)
+        self.db = self.conn[db_name].with_options(codec_options=codec_options)
         self.userdb = UserDb(self.db)
         self.actiondb = ActionDb(self.db)
         self.workerdb = WorkerDb(self.db)
@@ -149,8 +151,6 @@ def new_run(
     ):
         if start_time is None:
             start_time = datetime.now(timezone.utc)
-        else:
-            start_time = start_time.replace(tzinfo=timezone.utc)
 
         run_args = {
             "base_tag": base_tag,
@@ -464,10 +464,7 @@ def scavenge(self, run):
         task_id = -1
         for task in run["tasks"]:
             task_id += 1
-            if (
-                task["active"]
-                and task["last_updated"].replace(tzinfo=timezone.utc) < old
-            ):
+            if task["active"] and task["last_updated"] < old:
                 task["active"] = False
                 dead_task = True
                 print(
@@ -507,9 +504,7 @@ def get_machines(self):
             task["worker_info"]
             | {
                 "last_updated": (
-                    task["last_updated"].replace(tzinfo=timezone.utc)
-                    if task.get("last_updated")
-                    else None
+                    task["last_updated"] if task.get("last_updated") else None
                 ),
                 "run": run,
                 "task_id": task_id,
@@ -852,9 +847,7 @@ def priority(run):  # lower is better
         for task in active_tasks:
             task_name = worker_name(task["worker_info"], short=True)
             if my_name == task_name:
-                last_update = (
-                    now - task["last_updated"].replace(tzinfo=timezone.utc)
-                ).seconds
+                last_update = (now - task["last_updated"]).seconds
                 # 120 = period of heartbeat in worker.
                 if last_update <= 120:
                     error = 'Request_task: There is already a worker running with name "{}" which sent an update {} seconds ago'.format(
@@ -1406,10 +1399,7 @@ def purge_run(self, run, p=0.001, res=7.0, iters=1):
             return "Can only purge completed run"
 
         now = datetime.now(timezone.utc)
-        if (
-            "start_time" not in run
-            or (now - run["start_time"].replace(tzinfo=timezone.utc)).days > 30
-        ):
+        if "start_time" not in run or (now - run["start_time"]).days > 30:
             return "Run too old to be purged"
         # Do not revive failed runs
         if run.get("failed", False):

server/fishtest/schemas.py

@@ -82,9 +82,8 @@
 
 
 def first_test_before_last(x):
-    # Pymongo is not timezone aware. Assume dates are UTC.
-    f = x["first_test"]["date"].replace(tzinfo=timezone.utc)
-    l = x["last_test"]["date"].replace(tzinfo=timezone.utc)
+    f = x["first_test"]["date"]
+    l = x["last_test"]["date"]
     if f <= l:
         return True
     else:

server/fishtest/util.py

@@ -425,11 +425,11 @@ def post_in_fishcooking_results(run):
 
 
 def diff_date(date):
-    utc_date = date.replace(tzinfo=timezone.utc)
-    if utc_date != datetime.min.replace(tzinfo=timezone.utc):
-        diff = datetime.now(timezone.utc) - utc_date
-    else:
-        diff = timedelta.max
+    diff = (
+        datetime.now(timezone.utc) - date
+        if date != datetime.min.replace(tzinfo=timezone.utc)
+        else timedelta.max
+    )
     return diff
 
 

server/fishtest/views.py

@@ -837,10 +837,7 @@ def validate_modify(request, run):
         raise home(request)
 
     now = datetime.now(timezone.utc)
-    if (
-        "start_time" not in run
-        or (now - run["start_time"].replace(tzinfo=timezone.utc)).days > 30
-    ):
+    if "start_time" not in run or (now - run["start_time"]).days > 30:
         request.session.flash("Run too old to be modified", "error")
         raise home(request)
 
@@ -1565,8 +1562,8 @@ def tests_view(request):
         if task["active"]:
             active += 1
             cores += task["worker_info"]["concurrency"]
-        last_updated = task.get("last_updated", datetime.min).replace(
-            tzinfo=timezone.utc
+        last_updated = task.get(
+            "last_updated", datetime.min.replace(tzinfo=timezone.utc)
         )
         task["last_updated"] = last_updated
 

server/tests/test_nn.py

@@ -1,5 +1,5 @@
 import unittest
-from datetime import datetime
+from datetime import datetime, timezone
 
 from util import get_rundb
 from vtjson import ValidationError
@@ -15,9 +15,9 @@ def setUp(self):
         self.rundb = get_rundb()
         self.name = "nn-0000000000a0.nnue"
         self.user = "user00"
-        self.first_test = datetime(2024, 1, 1)
-        self.last_test = datetime(2024, 3, 24)
-        self.last_test_old = datetime(2023, 3, 24)
+        self.first_test = datetime(2024, 1, 1, tzinfo=timezone.utc)
+        self.last_test = datetime(2024, 3, 24, tzinfo=timezone.utc)
+        self.last_test_old = datetime(2023, 3, 24, tzinfo=timezone.utc)
         self.run_id = "64e74776a170cb1f26fa3930"
 
     def tearDown(self):

server/utils/delta_update_users.py

@@ -87,8 +87,8 @@ def process_run(run, info):
 
         info_user = info[t_username]
         info_user["last_updated"] = max(
-            info_user["last_updated"].replace(tzinfo=timezone.utc),
-            task.get("last_updated", datetime.min).replace(tzinfo=timezone.utc),
+            info_user["last_updated"],
+            task.get("last_updated", datetime.min.replace(tzinfo=timezone.utc)),
         )
         info_user["cpu_hours"] += float(
             num_games * int(run["args"].get("threads", 1)) * tc / (60 * 60)
@@ -131,7 +131,7 @@ def update_info(rundb, clear_stats, deltas, info_total, info_top_month):
             skip_count += 1
 
         # Update info_top_month with finished runs having start_time in the last 30 days
-        if (now - run["start_time"].replace(tzinfo=timezone.utc)).days < 30:
+        if (now - run["start_time"]).days < 30:
             try:
                 process_run(run, info_top_month)
             except Exception as e:
@@ -193,25 +193,26 @@ def update_users(rundb, users_total, users_top_month):
 def cleanup_users(rundb):
     # Delete users that have never been active and old admins group
     idle = {}
+    now = datetime.now(timezone.utc)
     for u in rundb.userdb.get_users():
         update = False
         while "group:admins" in u["groups"]:
             u["groups"].remove("group:admins")
             update = True
         if update:
             rundb.userdb.save_user(u)
-        if "registration_time" not in u or u["registration_time"].replace(
-            tzinfo=timezone.utc
-        ) < datetime.now(timezone.utc) - timedelta(days=28):
+        if "registration_time" not in u or u["registration_time"] < now - timedelta(
+            days=28
+        ):
             idle[u["username"]] = u
     for u in rundb.userdb.user_cache.find():
         if u["username"] in idle:
             del idle[u["username"]]
     for u in idle.values():
         # A safe guard against deleting long time users
-        if "registration_time" not in u or u["registration_time"].replace(
-            tzinfo=timezone.utc
-        ) < datetime.now(timezone.utc) - timedelta(days=38):
+        if "registration_time" not in u or u["registration_time"] < now - timedelta(
+            days=38
+        ):
             print("Warning: Found old user to delete:", str(u["_id"]))
         else:
             print("Delete:", str(u["_id"]))

server/utils/purge_pgn.py

@@ -23,8 +23,8 @@ def purge_pgn(rundb, finished, deleted, days):
             not deleted
             and finished
             and tc_regex.match(run["args"]["tc"])
-            and run["last_updated"].replace(tzinfo=timezone.utc) > cutoff_date_ltc
-        ) or run["last_updated"].replace(tzinfo=timezone.utc) > cutoff_date
+            and run["last_updated"] > cutoff_date_ltc
+        ) or run["last_updated"] > cutoff_date
 
         if keep:
             kept_runs += 1