Add filtering and heuristics

pcap-extractor/VERSION

@@ -1 +1 @@
-0
+1

pcap-extractor/service/al_run.py

@@ -18,11 +18,20 @@ def __init__(self, config=None):
 
     def _load_config(self):
         self.local_networks = []
+        self.ignore_ips = []
+
+        # Local - do not tag IPs from these networks
         local_networks = self.config.get("local_networks", "").split(",")
         if local_networks:
             for network in local_networks:
                 self.local_networks.append(ipaddress.ip_network(network))
 
+        # Ignore - ignore traffic to these IPs
+        ignore_ips = self.config.get("ignore_ips", "").split(",")
+        for ip in ignore_ips:
+            if ip:
+                self.ignore_ips.append(ipaddress.ip_address(ip))
+
         self.command_timeout = int(self.config.get("command_timeout", 30))
 
     def start(self):
@@ -52,18 +61,27 @@ def execute(self, request: ServiceRequest) -> None:
         main_section.add_subsection(tcp_section)
 
         extractor = Extractor(
-            request.file_path, base_logger=self.log, timeout=self.command_timeout
+            request.file_path,
+            base_logger=self.log,
+            timeout=self.command_timeout,
+            ignore_ips=self.ignore_ips,
         )
         for conv in extractor.process_conversations():
             protocol = "TCP" if not conv.is_http else "HTTP"
-            if not self._is_local_network(conv.src_ip):
+            source_local = self._is_local_network(conv.src_ip)
+            destination_local = self._is_local_network(conv.dst_ip)
+            if not source_local:
                 tcp_section.add_tag("network.dynamic.ip", conv.src_ip)
-            if not self._is_local_network(conv.dst_ip):
+            if not destination_local:
                 tcp_section.add_tag("network.dynamic.ip", conv.dst_ip)
 
             conversation_section = ResultTextSection(
                 f"{protocol} {conv.description}", auto_collapse=True
             )
+            if not conv.is_http:
+                conversation_section.set_heuristic(2)
+            elif not source_local or not destination_local:
+                conversation_section.set_heuristic(1)
 
             flow_section = ResultMemoryDumpSection("Data flow sample")
             flow_section.add_line(self._read_stream_sample(conv.stream_file))

pcap-extractor/service/extractor.py

@@ -17,6 +17,7 @@ class Conversation:
     dst_port: int
     stream_file: str = None
 
+    @property
     def is_http(self) -> bool:
         return self.dst_port in [80, 443] or self.src_port in [80, 443]
 
@@ -35,6 +36,7 @@ def __init__(
         pcap_path: str,
         base_logger: logging.Logger = None,
         timeout: int = 20,
+        ignore_ips: list[ipaddress._IPAddressBase] = None,
     ) -> None:
         self.pcap_path = pcap_path
         self.tshark_path = TSHARK_PATH
@@ -44,13 +46,27 @@ def __init__(
             else logging.getLogger(__name__)
         )
         self.timeout = timeout
+        self.ignore_ips = ignore_ips or []
+
+    def _ignored_filter(self) -> list[str]:
+        if not self.ignore_ips:
+            return []
+        return [
+            "-2",
+            "-R",
+            f"ip.addr not in {{{','.join([str(ip) for ip in self.ignore_ips])}}}",
+        ]
 
     def execute(self, command: list[str], out_file: str = None) -> str:
         kwargs = {}
         if out_file:
             kwargs["stdout"] = open(out_file, "w")
         result = subprocess.run(
-            [self.tshark_path, "-r", self.pcap_path, "-q"] + command,
+            (
+                [self.tshark_path, "-r", self.pcap_path, "-q"]
+                + self._ignored_filter()
+                + command
+            ),
             capture_output=True if not out_file else False,
             text=True,
             timeout=self.timeout,
@@ -99,7 +115,7 @@ def _extract_http_stream(self, conv: Conversation):
 
     def process_conversations(self) -> Iterable[Conversation]:
         for conv in self.get_tcp_conversations():
-            if conv.is_http():
+            if conv.is_http:
                 self._extract_http_stream(conv)
             else:
                 self._extract_tcp_stream(conv)

pcap-extractor/service_manifest.yml

@@ -13,6 +13,7 @@ is_external: false
 
 config:
   local_networks: "172.17.0.0/16"
+  ignore_ips: "192.168.0.1"
   command_timeout: 30
 
 # submission_params:
@@ -28,12 +29,17 @@ config:
 # 700 - 999: highly suspicious
 # >= 1000: malicious
 
-# heuristics:
-#   - description: Some score
-#     filetype: "*"
-#     heur_id: 1
-#     name: Score
-#     score: 0
+heuristics:
+  - description: Connection to external IP was found in the PCAP
+    filetype: "*"
+    heur_id: 1
+    name: External HTTP connection
+    score: 300
+  - description: Connection to non-standard port was found in the PCAP. Very suspicious.
+    filetype: "*"
+    heur_id: 2
+    name: Connection to non-standard port
+    score: 700
 
 docker_config:
   image: ${REGISTRY}kam193/assemblyline-service-pcap-extractor:4.4.0.stable$VERSION