Strip user-agent leading whitespaces and ignore empty user agents

openzim · Jun 3, 2024 · 0f39911 · 0f39911
1 parent 957e52c
commit 0f39911
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/src/zimit/zimit.py b/src/zimit/zimit.py
@@ -558,6 +558,12 @@ def get_node_cmd_line(args):
         "logging",
     ]:
         value = getattr(args, arg)
+        if arg == "userAgent":
+            # strip leading whitespace which are not allowed on some websites
+            value = value.strip()
+            if not value:
+                # ignore empty userAgent arg and keep crawler default value if empty
+                continue
         if value is None or (isinstance(value, bool) and value is False):
             continue
         node_cmd.append("--" + arg)