From 7c314ac62160c9b5acea23f029567d92ae8369fa Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 10:56:44 +0100
Subject: [PATCH 1/9] Netwatch example: bulk domain lists

---
 examples/livehunt_network_watch.py | 65 ++++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 12 deletions(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 5cce2b6..f2ed174 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -39,6 +39,11 @@
 RULESET_ENTITY = ("file", "url", "domain", "ip_address")
 RULESET_LINK = "https://www.virustotal.com/yara-editor/livehunt/"
 
+EMPTY_DOMAIN_LIST_MSG = (
+  "* Empty domain list, use --add-domain domain.tld or bulk operations to"
+  " register them"
+)
+
 
 def extract_domains_from_rule(rules):
   """Extract the domain list from the comment of a yara rule."""
@@ -148,6 +153,20 @@ async def upload_rulesets(queue):
       queue.task_done()
 
 
+def load_bulk_file_domains(filename):
+  if not os.path.isfile(filename):
+    print(f"Error: File {filename} does not exists.")
+    sys.exit(1)
+
+  domains = []
+  with open(filename, encoding="utf-8") as bulk_file:
+    for line in bulk_file.read().split('\n'):
+      if not line:
+        continue
+      domains.append(line)
+  return domains
+
+
 async def main():
   parser = argparse.ArgumentParser(
       description=(
@@ -160,9 +179,23 @@ async def main():
       action="store_true",
       help="List current monitored domains.",
   )
-  parser.add_argument("-a", "--add-domain", help="Add a domain to the list.")
   parser.add_argument(
-      "-d", "--delete-domain", help="Remove a domain from the list."
+      "-a",
+      "--add-domain",
+      help="Add a domain to the list.",
+  )
+  parser.add_argument(
+      "-d",
+      "--delete-domain",
+      help="Remove a domain from the list.",
+  )
+  parser.add_argument(
+      "--bulk-append",
+      help="Add a list of domains from an input file.",
+  )
+  parser.add_argument(
+      "--bulk-replace",
+      help="Remove the remote list with a new list from a file.",
   )
   parser.add_argument(
       "--workers",
@@ -185,14 +218,15 @@ async def main():
     return
 
   rulesets = await get_rulesets()
-  if not rulesets and not args.add_domain:
-    print("* Empty domain list, use --add-domain domain.tld to register one")
+  if (not rulesets and
+      not (args.add_domain or args.bulk_append or args.bulk_replace)):
+    print(EMPTY_DOMAIN_LIST_MSG)
     sys.exit(1)
 
   domains = rulesets.get("url", {}).get("domains", [])
   if args.list:
     if not domains:
-      print("* Empty domain list, use --add-domain domain.tld to register one")
+      print(EMPTY_DOMAIN_LIST_MSG)
       sys.exit(0)
 
     print("Currently monitored domains:")
@@ -201,14 +235,21 @@ async def main():
     sys.exit(0)
 
   new_domain_list = copy.copy(domains)
-  if args.add_domain:
-    new_domain_list.append(args.add_domain)
+  if args.bulk_replace:
+    new_domain_list = load_bulk_file_domains(args.bulk_replace)
 
-  if args.delete_domain:
-    if not args.delete_domain in new_domain_list:
-      print(f"* {args.delete_domain} not in list")
-      sys.exit(1)
-    new_domain_list.remove(args.delete_domain)
+  elif args.bulk_append:
+    new_domain_list += load_bulk_file_domains(args.bulk_append)
+
+  else:
+    if args.add_domain:
+      new_domain_list.append(args.add_domain)
+
+    if args.delete_domain:
+      if not args.delete_domain in new_domain_list:
+        print(f"* {args.delete_domain} not in list")
+        sys.exit(1)
+      new_domain_list.remove(args.delete_domain)
 
   new_domain_list = list(set(new_domain_list))
   new_domain_list.sort()

From 6e58600fd0f8ec6b2e41534643866871ae99fde2 Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 10:58:09 +0100
Subject: [PATCH 2/9] typo

---
 examples/livehunt_network_watch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index f2ed174..343edb2 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -195,7 +195,7 @@ async def main():
   )
   parser.add_argument(
       "--bulk-replace",
-      help="Remove the remote list with a new list from a file.",
+      help="Replace the remote list with a new list from a file.",
   )
   parser.add_argument(
       "--workers",

From 2693c24b11490560db44f52a8c1cf8bd1aaac9f7 Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 11:39:48 +0100
Subject: [PATCH 3/9] as lists and fix rule

---
 examples/livehunt_network_watch.py          | 31 +++++++++++++++------
 examples/netwatch_templates/domain.yara     |  2 +-
 examples/netwatch_templates/file.yara       |  4 +--
 examples/netwatch_templates/ip_address.yara |  2 +-
 examples/netwatch_templates/url.yara        |  2 +-
 5 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 343edb2..8dc484f 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -123,14 +123,19 @@ async def upload_rulesets(queue):
         )
         try:
           # Fix for https://github.com/VirusTotal/vt-py/issues/155 issue.
-          await client.patch_async(
+          result = await client.patch_async(
               path="/intelligence/hunting_rulesets/" + task.get("id"),
               json_data={"data": ruleset.to_dict()},
           )
-          print(f'Ruleset {name} [{RULESET_LINK}{task["id"]}] updated.')
         except vt.error.APIError as e:
           print(f"Error updating {name}: {e}")
 
+        response = await result.json_async()
+        if response.get('error') != None:
+          print(f'{name}: {response}')
+
+        print(f'Ruleset {name} [{RULESET_LINK}{task["id"]}] updated.')
+
       else:
         ruleset = vt.Object(
             obj_type="hunting_ruleset",
@@ -146,10 +151,15 @@ async def upload_rulesets(queue):
           result = await client.post_object_async(
               path="/intelligence/hunting_rulesets", obj=ruleset
           )
-          print(f"Ruleset {name} [{RULESET_LINK}{result.id}] created.")
         except vt.error.APIError as e:
           print(f"Error saving {name}: {e}")
 
+        response = await result.json_async()
+        if response.get('error') != None:
+          print(f'{name}: {response}')
+
+        print(f"Ruleset {name} [{RULESET_LINK}{result.id}] created.")
+
       queue.task_done()
 
 
@@ -182,11 +192,15 @@ async def main():
   parser.add_argument(
       "-a",
       "--add-domain",
+      action="append",
+      type=str,
       help="Add a domain to the list.",
   )
   parser.add_argument(
       "-d",
       "--delete-domain",
+      action="append",
+      type=str,
       help="Remove a domain from the list.",
   )
   parser.add_argument(
@@ -243,13 +257,14 @@ async def main():
 
   else:
     if args.add_domain:
-      new_domain_list.append(args.add_domain)
+      new_domain_list += args.add_domain
 
     if args.delete_domain:
-      if not args.delete_domain in new_domain_list:
-        print(f"* {args.delete_domain} not in list")
-        sys.exit(1)
-      new_domain_list.remove(args.delete_domain)
+      for deleted_domain in args.delete_domain:
+        if not deleted_domain in new_domain_list:
+          print(f"* {deleted_domain} not in list")
+          sys.exit(1)
+        new_domain_list.remove(deleted_domain)
 
   new_domain_list = list(set(new_domain_list))
   new_domain_list.sort()
diff --git a/examples/netwatch_templates/domain.yara b/examples/netwatch_templates/domain.yara
index 67a131e..e6500bf 100644
--- a/examples/netwatch_templates/domain.yara
+++ b/examples/netwatch_templates/domain.yara
@@ -1,4 +1,4 @@
-rule network_watch_${domain_escaped} : ${domain_escaped} {
+rule network_watch_${domain_escaped} : domain_${domain_escaped} {
 meta:
   description = "Monitor new subdomains for ${domain}"
   target_entity = "domain"
diff --git a/examples/netwatch_templates/file.yara b/examples/netwatch_templates/file.yara
index b32ef09..831acbc 100644
--- a/examples/netwatch_templates/file.yara
+++ b/examples/netwatch_templates/file.yara
@@ -1,4 +1,4 @@
-rule network_watch_${domain_escaped} : ${domain_escaped} {
+rule network_watch_${domain_escaped} : domain_${domain_escaped} {
 meta:
   description = "New files downloaded from ${domain}"
   target_entity = "file"
@@ -8,7 +8,7 @@ condition:
 }
 
 
-rule network_watch_contact_${domain_escaped} : ${domain_escaped} {
+rule network_watch_contact_${domain_escaped} : domain_${domain_escaped} {
 meta:
   description = "New files contacting ${domain}"
   target_entity = "file"
diff --git a/examples/netwatch_templates/ip_address.yara b/examples/netwatch_templates/ip_address.yara
index d3bf76b..e2d06a2 100644
--- a/examples/netwatch_templates/ip_address.yara
+++ b/examples/netwatch_templates/ip_address.yara
@@ -1,4 +1,4 @@
-rule network_watch_${domain_escaped} : ${domain_escaped} {
+rule network_watch_${domain_escaped} : domain_${domain_escaped} {
 meta:
   description = "New IP addresses resolving domain ${domain} or its subdomains"
   target_entity = "ip_address"
diff --git a/examples/netwatch_templates/url.yara b/examples/netwatch_templates/url.yara
index 75863d9..3ac2099 100644
--- a/examples/netwatch_templates/url.yara
+++ b/examples/netwatch_templates/url.yara
@@ -1,4 +1,4 @@
-rule network_watch_${domain_escaped} : ${domain_escaped} {
+rule network_watch_${domain_escaped} : domain_${domain_escaped} {
 meta:
   description = "Monitor new URLs in ${domain}"
   target_entity = "url"

From fd8199d2b99587bbc0d41bba02395318b898df75 Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 12:41:32 +0100
Subject: [PATCH 4/9] fix potential dupes

---
 examples/livehunt_network_watch.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 8dc484f..e8f3fda 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -85,11 +85,21 @@ def render_template(entity, domains):
     template += "\n"
 
   kind_template = os.path.join(TEMPLATE_DIR, entity + ".yara")
+  escaped_domains = {}
   with open(kind_template, encoding="utf-8") as f:
     rule_block = f.read()
 
     for domain in domains:
-      domain_escaped = re.compile(r"[^\w\d]").sub("_", domain)
+      domain_escaped = re.compile(r"[^[a-z\d]").sub("_", domain)
+      domain_escaped = re.compile(r'(_(?i:_)+)').sub('_', domain_escaped)
+
+      if not domain_escaped in escaped_domains:
+        escaped_domains[domain_escaped] = 0
+      escaped_domains[domain_escaped] += 1
+
+      if escaped_domains[domain_escaped] > 1:
+        domain_escaped = f"{domain_escaped}_{escaped_domains[domain_escaped]}"
+
       template += rule_block.replace("${domain}", domain).replace(
           "${domain_escaped}", domain_escaped
       )

From 93bb658a3f730eead772ee7aa771180d8af88789 Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 12:46:58 +0100
Subject: [PATCH 5/9] lint

---
 examples/livehunt_network_watch.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index e8f3fda..54c1f8c 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -91,7 +91,7 @@ def render_template(entity, domains):
 
     for domain in domains:
       domain_escaped = re.compile(r"[^[a-z\d]").sub("_", domain)
-      domain_escaped = re.compile(r'(_(?i:_)+)').sub('_', domain_escaped)
+      domain_escaped = re.compile(r"(_(?i:_)+)").sub("_", domain_escaped)
 
       if not domain_escaped in escaped_domains:
         escaped_domains[domain_escaped] = 0
@@ -141,7 +141,7 @@ async def upload_rulesets(queue):
           print(f"Error updating {name}: {e}")
 
         response = await result.json_async()
-        if response.get('error') != None:
+        if response.get("error") is not Non:
           print(f'{name}: {response}')
 
         print(f'Ruleset {name} [{RULESET_LINK}{task["id"]}] updated.')
@@ -165,7 +165,7 @@ async def upload_rulesets(queue):
           print(f"Error saving {name}: {e}")
 
         response = await result.json_async()
-        if response.get('error') != None:
+        if response.get("error") is not Non:
           print(f'{name}: {response}')
 
         print(f"Ruleset {name} [{RULESET_LINK}{result.id}] created.")
@@ -180,7 +180,7 @@ def load_bulk_file_domains(filename):
 
   domains = []
   with open(filename, encoding="utf-8") as bulk_file:
-    for line in bulk_file.read().split('\n'):
+    for line in bulk_file.read().split("\n"):
       if not line:
         continue
       domains.append(line)

From f1f121a28985a02edfbee49d09a889c2f55a4e13 Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 13:00:58 +0100
Subject: [PATCH 6/9] lint+

---
 examples/livehunt_network_watch.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 54c1f8c..3a32d03 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -141,8 +141,8 @@ async def upload_rulesets(queue):
           print(f"Error updating {name}: {e}")
 
         response = await result.json_async()
-        if response.get("error") is not Non:
-          print(f'{name}: {response}')
+        if response.get("error") is not None:
+          print(f"{name}: {response}")
 
         print(f'Ruleset {name} [{RULESET_LINK}{task["id"]}] updated.')
 
@@ -165,8 +165,8 @@ async def upload_rulesets(queue):
           print(f"Error saving {name}: {e}")
 
         response = await result.json_async()
-        if response.get("error") is not Non:
-          print(f'{name}: {response}')
+        if response.get("error") is not None:
+          print(f"{name}: {response}")
 
         print(f"Ruleset {name} [{RULESET_LINK}{result.id}] created.")
 

From 9bac16904d5a57e7479a0fdcd9cd0277f0fefb48 Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 13:58:12 +0100
Subject: [PATCH 7/9] lower

---
 examples/livehunt_network_watch.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 3a32d03..2ca212f 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -90,7 +90,8 @@ def render_template(entity, domains):
     rule_block = f.read()
 
     for domain in domains:
-      domain_escaped = re.compile(r"[^[a-z\d]").sub("_", domain)
+      domain_escaped = domain.lower()
+      domain_escaped = re.compile(r"[^[a-z\d]").sub("_", domain_escaped)
       domain_escaped = re.compile(r"(_(?i:_)+)").sub("_", domain_escaped)
 
       if not domain_escaped in escaped_domains:
@@ -277,6 +278,7 @@ async def main():
         new_domain_list.remove(deleted_domain)
 
   new_domain_list = list(set(new_domain_list))
+  new_domain_list = [domain.lower() for domain in new_domain_list]
   new_domain_list.sort()
 
   if new_domain_list != domains:

From 5664a92b7f04f80aa8d75689a247beaa9efde10a Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 14:00:28 +0100
Subject: [PATCH 8/9] terminate

---
 examples/livehunt_network_watch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 2ca212f..5870d20 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -140,6 +140,7 @@ async def upload_rulesets(queue):
           )
         except vt.error.APIError as e:
           print(f"Error updating {name}: {e}")
+          sys.exit(1)
 
         response = await result.json_async()
         if response.get("error") is not None:
@@ -164,6 +165,7 @@ async def upload_rulesets(queue):
           )
         except vt.error.APIError as e:
           print(f"Error saving {name}: {e}")
+          sys.exit(1)
 
         response = await result.json_async()
         if response.get("error") is not None:

From db10ca6b03c2213026914ec3ad79652cee60e1eb Mon Sep 17 00:00:00 2001
From: Francisco Santos <fsantos@virustotal.com>
Date: Fri, 24 Nov 2023 14:21:31 +0100
Subject: [PATCH 9/9] exits

---
 examples/livehunt_network_watch.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/livehunt_network_watch.py b/examples/livehunt_network_watch.py
index 5870d20..3a4f8c1 100644
--- a/examples/livehunt_network_watch.py
+++ b/examples/livehunt_network_watch.py
@@ -145,6 +145,7 @@ async def upload_rulesets(queue):
         response = await result.json_async()
         if response.get("error") is not None:
           print(f"{name}: {response}")
+          sys.exit(1)
 
         print(f'Ruleset {name} [{RULESET_LINK}{task["id"]}] updated.')
 
@@ -170,6 +171,7 @@ async def upload_rulesets(queue):
         response = await result.json_async()
         if response.get("error") is not None:
           print(f"{name}: {response}")
+          sys.exit(1)
 
         print(f"Ruleset {name} [{RULESET_LINK}{result.id}] created.")