Merge branch 'main' into fix/strip-translated-file-path

dissect/target/plugins/os/unix/linux/network_managers.py

@@ -567,7 +567,7 @@ def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord |
             continue
 
         # Debian and CentOS dhclient
-        if hasattr(record, "daemon") and record.daemon == "dhclient" and "bound to" in line:
+        if hasattr(record, "service") and record.service == "dhclient" and "bound to" in line:
             ip = line.split("bound to")[1].split(" ")[1].strip()
             ips.add(ip)
             continue

dissect/target/plugins/os/unix/log/auth.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import itertools
 import logging
 import re
 from abc import ABC, abstractmethod
@@ -12,24 +11,18 @@
 
 from dissect.target import Target
 from dissect.target.exceptions import UnsupportedPluginError
-from dissect.target.helpers.fsutil import open_decompress
 from dissect.target.helpers.record import DynamicDescriptor, TargetRecordDescriptor
 from dissect.target.helpers.utils import year_rollover_helper
 from dissect.target.plugin import Plugin, alias, export
+from dissect.target.plugins.os.unix.log.helpers import (
+    RE_LINE,
+    RE_TS,
+    is_iso_fmt,
+    iso_readlines,
+)
 
 log = logging.getLogger(__name__)
 
-RE_TS = re.compile(r"^[A-Za-z]{3}\s*\d{1,2}\s\d{1,2}:\d{2}:\d{2}")
-RE_TS_ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}")
-RE_LINE = re.compile(
-    r"""
-    \d{2}:\d{2}\s                           # First match on the similar ending of the different timestamps
-    (?P<hostname>\S+)\s                     # The hostname
-    (?P<service>\S+?)(\[(?P<pid>\d+)\])?:   # The service with optionally the PID between brackets
-    \s*(?P<message>.+?)\s*$                 # The log message stripped from spaces left and right
-    """,
-    re.VERBOSE,
-)
 
 # Generic regular expressions
 RE_IPV4_ADDRESS = re.compile(
@@ -347,27 +340,3 @@ def authlog(self) -> Iterator[Any]:
 
             for ts, line in iterable:
                 yield self._auth_log_builder.build_record(ts, auth_file, line)
-
-
-def iso_readlines(file: Path) -> Iterator[tuple[datetime, str]]:
-    """Iterator reading the provided auth log file in ISO format. Mimics ``year_rollover_helper`` behaviour."""
-    with open_decompress(file, "rt") as fh:
-        for line in fh:
-            if not (match := RE_TS_ISO.match(line)):
-                log.warning("No timestamp found in one of the lines in %s!", file)
-                log.debug("Skipping line: %s", line)
-                continue
-
-            try:
-                ts = datetime.strptime(match[0], "%Y-%m-%dT%H:%M:%S.%f%z")
-            except ValueError as e:
-                log.warning("Unable to parse ISO timestamp in line: %s", line)
-                log.debug("", exc_info=e)
-                continue
-
-            yield ts, line
-
-
-def is_iso_fmt(file: Path) -> bool:
-    """Determine if the provided auth log file uses new ISO format logging or not."""
-    return any(itertools.islice(iso_readlines(file), 0, 2))

dissect/target/plugins/os/unix/log/helpers.py

@@ -0,0 +1,46 @@
+import itertools
+import logging
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Iterator
+
+from dissect.target.helpers.fsutil import open_decompress
+
+log = logging.getLogger(__name__)
+
+RE_TS = re.compile(r"^[A-Za-z]{3}\s*\d{1,2}\s\d{1,2}:\d{2}:\d{2}")
+RE_TS_ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}")
+RE_LINE = re.compile(
+    r"""
+    \d{2}:\d{2}\s                           # First match on the similar ending of the different timestamps
+    (?:\S+)\s                               # The hostname, but do not capture it
+    (?P<service>\S+?)(\[(?P<pid>\d+)\])?:    # The service / daemon with optionally the PID between brackets
+    \s*(?P<message>.+?)\s*$                 # The log message stripped from spaces left and right
+    """,
+    re.VERBOSE,
+)
+
+
+def iso_readlines(file: Path) -> Iterator[tuple[datetime, str]]:
+    """Iterator reading the provided log file in ISO format. Mimics ``year_rollover_helper`` behaviour."""
+    with open_decompress(file, "rt") as fh:
+        for line in fh:
+            if not (match := RE_TS_ISO.match(line)):
+                log.warning("No timestamp found in one of the lines in %s!", file)
+                log.debug("Skipping line: %s", line)
+                continue
+
+            try:
+                ts = datetime.strptime(match[0], "%Y-%m-%dT%H:%M:%S.%f%z")
+            except ValueError as e:
+                log.warning("Unable to parse ISO timestamp in line: %s", line)
+                log.debug("", exc_info=e)
+                continue
+
+            yield ts, line
+
+
+def is_iso_fmt(file: Path) -> bool:
+    """Determine if the provided log file uses ISO 8601 timestamp format logging or not."""
+    return any(itertools.islice(iso_readlines(file), 0, 2))

dissect/target/plugins/os/unix/log/messages.py

@@ -11,25 +11,27 @@
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.helpers.utils import year_rollover_helper
 from dissect.target.plugin import Plugin, alias, export
+from dissect.target.plugins.os.unix.log.helpers import (
+    RE_LINE,
+    RE_TS,
+    is_iso_fmt,
+    iso_readlines,
+)
 
 MessagesRecord = TargetRecordDescriptor(
     "linux/log/messages",
     [
         ("datetime", "ts"),
-        ("string", "daemon"),
+        ("string", "service"),
         ("varint", "pid"),
         ("string", "message"),
         ("path", "source"),
     ],
 )
 
 DEFAULT_TS_LOG_FORMAT = "%b %d %H:%M:%S"
-RE_TS = re.compile(r"(\w+\s{1,2}\d+\s\d{2}:\d{2}:\d{2})")
-RE_DAEMON = re.compile(r"^[^:]+:\d+:\d+[^\[\]:]+\s([^\[:]+)[\[|:]{1}")
-RE_PID = re.compile(r"\w\[(\d+)\]")
-RE_MSG = re.compile(r"[^:]+:\d+:\d+[^:]+:\s(.*)$")
 RE_CLOUD_INIT_LINE = re.compile(
-    r"^(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) - (?P<daemon>.*)\[(?P<log_level>\w+)\]\: (?P<message>.*)$"
+    r"^(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) - (?P<service>.*)\[(?P<log_level>\w+)\]\: (?P<message>.*)$"
 )
 
 
@@ -56,7 +58,7 @@ def check_compatible(self) -> None:
     def messages(self) -> Iterator[MessagesRecord]:
         """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs.
 
-        Due to year rollover detection, the contents of the files are returned in reverse.
+        Due to year rollover detection, the log contents could be returned in reversed or mixed chronological order.
 
         The messages log file holds information about a variety of events such as the system error messages, system
         startups and shutdowns, change in the network configuration, etc. Aims to store valuable, non-debug and
@@ -75,16 +77,23 @@ def messages(self) -> Iterator[MessagesRecord]:
                 yield from self._parse_cloud_init_log(log_file, tzinfo)
                 continue
 
-            for ts, line in year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo):
-                daemon = dict(enumerate(RE_DAEMON.findall(line))).get(0)
-                pid = dict(enumerate(RE_PID.findall(line))).get(0)
-                message = dict(enumerate(RE_MSG.findall(line))).get(0, line)
+            if is_iso_fmt(log_file):
+                iterable = iso_readlines(log_file)
+
+            else:
+                iterable = year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo)
+
+            for ts, line in iterable:
+                match = RE_LINE.search(line)
+
+                if not match:
+                    self.target.log.warning("Unable to parse message line in %s", log_file)
+                    self.target.log.debug("Line %s", line)
+                    continue
 
                 yield MessagesRecord(
                     ts=ts,
-                    daemon=daemon,
-                    pid=pid,
-                    message=message,
+                    **match.groupdict(),
                     source=log_file,
                     _target=self.target,
                 )
@@ -134,7 +143,7 @@ def _parse_cloud_init_log(self, log_file: Path, tzinfo: tzinfo | None = timezone
 
                 yield MessagesRecord(
                     ts=ts,
-                    daemon=values["daemon"],
+                    service=values["service"],
                     pid=None,
                     message=values["message"],
                     source=log_file,

dissect/target/plugins/os/windows/catroot.py

@@ -227,8 +227,14 @@ def catdb(self) -> Iterator[CatrootRecord]:
 
                     for record in table.records():
                         file_digest = digest()
-                        setattr(file_digest, hash_type, record.get("HashCatNameTable_HashCol").hex())
-                        catroot_names = record.get("HashCatNameTable_CatNameCol").decode().rstrip("|").split("|")
+
+                        try:
+                            setattr(file_digest, hash_type, record.get("HashCatNameTable_HashCol").hex())
+                            catroot_names = record.get("HashCatNameTable_CatNameCol").decode().rstrip("|").split("|")
+                        except Exception as e:
+                            self.target.log.warning("Unable to parse catroot names for %s in %s", record, ese_file)
+                            self.target.log.debug("", exc_info=e)
+                            continue
 
                         for catroot_name in catroot_names:
                             yield CatrootRecord(

dissect/target/plugins/os/windows/lnk.py

@@ -1,4 +1,6 @@
-from typing import Iterator, Optional
+from __future__ import annotations
+
+from typing import Iterator
 
 from dissect.shellitem.lnk import Lnk
 from dissect.util import ts
@@ -34,7 +36,7 @@
 )
 
 
-def parse_lnk_file(target: Target, lnk_file: Lnk, lnk_path: TargetPath) -> Iterator[LnkRecord]:
+def parse_lnk_file(target: Target, lnk_file: Lnk, lnk_path: TargetPath) -> LnkRecord:
     # we need to get the active codepage from the system to properly decode some values
     codepage = target.codepage or "ascii"
 
@@ -132,7 +134,7 @@ def check_compatible(self) -> None:
 
     @arg("--path", "-p", dest="path", default=None, help="Path to directory or .lnk file in target")
     @export(record=LnkRecord)
-    def lnk(self, path: Optional[str] = None) -> Iterator[LnkRecord]:
+    def lnk(self, path: str | None = None) -> Iterator[LnkRecord]:
         """Parse all .lnk files in /ProgramData, /Users, and /Windows or from a specified path in record format.
 
         Yields a LnkRecord record with the following fields:
@@ -160,10 +162,14 @@ def lnk(self, path: Optional[str] = None) -> Iterator[LnkRecord]:
         """
 
         for entry in self.lnk_entries(path):
-            lnk_file = Lnk(entry.open())
-            yield parse_lnk_file(self.target, lnk_file, entry)
-
-    def lnk_entries(self, path: Optional[str] = None) -> Iterator[TargetPath]:
+            try:
+                lnk_file = Lnk(entry.open())
+                yield parse_lnk_file(self.target, lnk_file, entry)
+            except Exception as e:
+                self.target.log.warning("Failed to parse link file %s", lnk_file)
+                self.target.log.debug("", exc_info=e)
+
+    def lnk_entries(self, path: str | None = None) -> Iterator[TargetPath]:
         if path:
             target_path = self.target.fs.path(path)
             if not target_path.exists():

dissect/target/plugins/os/windows/regf/cit.py

@@ -632,16 +632,16 @@ def local_wintimestamp(target, ts):
 class CITPlugin(Plugin):
     """Plugin that parses CIT data from the registry.
 
-    Reference:
-    - https://dfir.ru/2018/12/02/the-cit-database-and-the-syscache-hive/
+    References:
+        - https://dfir.ru/2018/12/02/the-cit-database-and-the-syscache-hive/
     """
 
     __namespace__ = "cit"
 
     KEY = "HKLM\\Software\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\CIT"
 
     def check_compatible(self) -> None:
-        if not len(list(self.target.registry.keys(self.KEY))) > 0:
+        if not list(self.target.registry.keys(self.KEY)):
             raise UnsupportedPluginError("No CIT registry key found")
 
     @export(record=get_args(CITRecords))
@@ -770,8 +770,9 @@ def cit(self) -> Iterator[CITRecords]:
                         yield from _yield_bitmap_records(
                             self.target, cit, entry.use_data.bitmaps.foreground, CITProgramBitmapForegroundRecord
                         )
-                except Exception:
-                    self.target.log.exception("Failed to parse CIT value: %s", value.name)
+                except Exception as e:
+                    self.target.log.warning("Failed to parse CIT value: %s", value.name)
+                    self.target.log.debug("", exc_info=e)
 
     @export(record=CITPostUpdateUseInfoRecord)
     def puu(self) -> Iterator[CITPostUpdateUseInfoRecord]:
@@ -788,10 +789,16 @@ def puu(self) -> Iterator[CITPostUpdateUseInfoRecord]:
         for reg_key in keys:
             for key in self.target.registry.keys(reg_key):
                 try:
-                    puu = c_cit.CIT_POST_UPDATE_USE_INFO(key.value("PUUActive").value)
+                    key_value = key.value("PUUActive").value
+                    puu = c_cit.CIT_POST_UPDATE_USE_INFO(key_value)
                 except RegistryValueNotFoundError:
                     continue
 
+                except EOFError as e:
+                    self.target.log.warning("Exception reading CIT structure in key %s", key.path)
+                    self.target.log.debug("Unable to parse value %s", key_value, exc_info=e)
+                    continue
+
                 yield CITPostUpdateUseInfoRecord(
                     log_time_start=wintimestamp(puu.LogTimeStart),
                     update_key=puu.UpdateKey,
@@ -852,10 +859,16 @@ def dp(self) -> Iterator[CITDPRecord | CITDPDurationRecord]:
         for reg_key in keys:
             for key in self.target.registry.keys(reg_key):
                 try:
-                    dp = c_cit.CIT_DP_DATA(key.value("DP").value)
+                    key_value = key.value("DP").value
+                    dp = c_cit.CIT_DP_DATA(key_value)
                 except RegistryValueNotFoundError:
                     continue
 
+                except EOFError as e:
+                    self.target.log.warning("Exception reading CIT structure in key %s", key.path)
+                    self.target.log.debug("Unable to parse value %s", key_value, exc_info=e)
+                    continue
+
                 user = self.target.registry.get_user(key)
                 log_time_start = wintimestamp(dp.LogTimeStart)