Merge branch 'main' into fix/velociraptor_zip

dissect/target/helpers/regutil.py

@@ -12,7 +12,7 @@
 from pathlib import Path
 from typing import BinaryIO, Iterator, Optional, TextIO, Union
 
-from dissect.regf import regf
+from dissect.regf import c_regf, regf
 
 from dissect.target.exceptions import (
     RegistryError,
@@ -31,16 +31,33 @@
 
 
 class RegistryValueType(IntEnum):
-    NONE = regf.REG_NONE
-    SZ = regf.REG_SZ
-    EXPAND_SZ = regf.REG_EXPAND_SZ
-    BINARY = regf.REG_BINARY
-    DWORD = regf.REG_DWORD
-    DWORD_BIG_ENDIAN = regf.REG_DWORD_BIG_ENDIAN
-    MULTI_SZ = regf.REG_MULTI_SZ
-    FULL_RESOURCE_DESCRIPTOR = regf.REG_FULL_RESOURCE_DESCRIPTOR
-    RESOURCE_REQUIREMENTS_LIST = regf.REG_RESOURCE_REQUIREMENTS_LIST
-    QWORD = regf.REG_QWORD
+    """Registry value types as defined in ``winnt.h``.
+
+    Resources:
+        - https://learn.microsoft.com/en-us/windows/win32/sysinfo/registry-value-types
+        - https://github.com/fox-it/dissect.regf/blob/main/dissect/regf/c_regf.py
+    """
+
+    NONE = c_regf.REG_NONE
+    SZ = c_regf.REG_SZ
+    EXPAND_SZ = c_regf.REG_EXPAND_SZ
+    BINARY = c_regf.REG_BINARY
+    DWORD = c_regf.REG_DWORD
+    DWORD_BIG_ENDIAN = c_regf.REG_DWORD_BIG_ENDIAN
+    LINK = c_regf.REG_LINK
+    MULTI_SZ = c_regf.REG_MULTI_SZ
+    RESOURCE_LIST = c_regf.REG_RESOURCE_LIST
+    FULL_RESOURCE_DESCRIPTOR = c_regf.REG_FULL_RESOURCE_DESCRIPTOR
+    RESOURCE_REQUIREMENTS_LIST = c_regf.REG_RESOURCE_REQUIREMENTS_LIST
+    QWORD = c_regf.REG_QWORD
+
+    @classmethod
+    def _missing_(cls, value: int) -> IntEnum:
+        # Allow values other than defined members
+        member = int.__new__(cls, value)
+        member._name_ = None
+        member._value_ = value
+        return member
 
 
 class RegistryHive:

dissect/target/loaders/itunes.py

@@ -163,8 +163,10 @@ def derive_key(self, password: str) -> bytes:
 
     def files(self) -> Iterator[FileInfo]:
         """Iterate all the files in this backup."""
-        for row in self.manifest_db.table("Files").rows():
-            yield FileInfo(self, row.fileID, row.domain, row.relativePath, row.flags, row.file)
+
+        if table := self.manifest_db.table("Files"):
+            for row in table.rows():
+                yield FileInfo(self, row.fileID, row.domain, row.relativePath, row.flags, row.file)
 
 
 class FileInfo:

dissect/target/plugins/apps/browser/iexplore.py

@@ -36,14 +36,18 @@ def find_containers(self, name: str) -> Iterator[table.Table]:
             All ``ContainerId`` values for the requested container name.
         """
         try:
-            for container_record in self.db.table("Containers").records():
+            table = self.db.table("Containers")
+
+            for container_record in table.records():
                 if record_name := container_record.get("Name"):
                     record_name = record_name.rstrip("\00").lower()
                     if record_name == name.lower():
                         container_id = container_record.get("ContainerId")
                         yield self.db.table(f"Container_{container_id}")
-        except KeyError:
-            pass
+
+        except KeyError as e:
+            self.target.log.warning("Exception while parsing EseDB Containers table")
+            self.target.log.debug("", exc_info=e)
 
     def _iter_records(self, name: str) -> Iterator[record.Record]:
         """Yield records from a Webcache container.

dissect/target/plugins/os/unix/esxi/_os.py

@@ -472,37 +472,39 @@ def parse_config_store(fh: BinaryIO) -> dict[str, Any]:
     db = sqlite3.SQLite3(fh)
 
     store = {}
-    for row in db.table("Config").rows():
-        component_name = row.Component
-        config_group_name = row.ConfigGroup
-        value_group_name = row.Name
-        identifier_name = row.Identifier
-
-        if component_name not in store:
-            store[component_name] = {}
-        component = store[component_name]
-
-        if config_group_name not in component:
-            component[config_group_name] = {}
-        config_group = component[config_group_name]
-
-        if value_group_name not in config_group:
-            config_group[value_group_name] = {}
-        value_group = config_group[value_group_name]
-
-        if identifier_name not in value_group:
-            value_group[identifier_name] = {}
-        identifier = value_group[identifier_name]
-
-        identifier["modified_time"] = row.ModifiedTime
-        identifier["creation_time"] = row.CreationTime
-        identifier["version"] = row.Version
-        identifier["success"] = row.Success
-        identifier["auto_conf_value"] = json.loads(row.AutoConfValue) if row.AutoConfValue else None
-        identifier["user_value"] = json.loads(row.UserValue) if row.UserValue else None
-        identifier["vital_value"] = json.loads(row.VitalValue) if row.VitalValue else None
-        identifier["cached_value"] = json.loads(row.CachedValue) if row.CachedValue else None
-        identifier["desired_value"] = json.loads(row.DesiredValue) if row.DesiredValue else None
-        identifier["revision"] = row.Revision
+
+    if table := db.table("Config"):
+        for row in table.rows():
+            component_name = row.Component
+            config_group_name = row.ConfigGroup
+            value_group_name = row.Name
+            identifier_name = row.Identifier
+
+            if component_name not in store:
+                store[component_name] = {}
+            component = store[component_name]
+
+            if config_group_name not in component:
+                component[config_group_name] = {}
+            config_group = component[config_group_name]
+
+            if value_group_name not in config_group:
+                config_group[value_group_name] = {}
+            value_group = config_group[value_group_name]
+
+            if identifier_name not in value_group:
+                value_group[identifier_name] = {}
+            identifier = value_group[identifier_name]
+
+            identifier["modified_time"] = row.ModifiedTime
+            identifier["creation_time"] = row.CreationTime
+            identifier["version"] = row.Version
+            identifier["success"] = row.Success
+            identifier["auto_conf_value"] = json.loads(row.AutoConfValue) if row.AutoConfValue else None
+            identifier["user_value"] = json.loads(row.UserValue) if row.UserValue else None
+            identifier["vital_value"] = json.loads(row.VitalValue) if row.VitalValue else None
+            identifier["cached_value"] = json.loads(row.CachedValue) if row.CachedValue else None
+            identifier["desired_value"] = json.loads(row.DesiredValue) if row.DesiredValue else None
+            identifier["revision"] = row.Revision
 
     return store

dissect/target/plugins/os/unix/linux/network_managers.py

@@ -567,7 +567,7 @@ def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord |
             continue
 
         # Debian and CentOS dhclient
-        if hasattr(record, "daemon") and record.daemon == "dhclient" and "bound to" in line:
+        if hasattr(record, "service") and record.service == "dhclient" and "bound to" in line:
             ip = line.split("bound to")[1].split(" ")[1].strip()
             ips.add(ip)
             continue

dissect/target/plugins/os/unix/log/auth.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import itertools
 import logging
 import re
 from abc import ABC, abstractmethod
@@ -12,24 +11,18 @@
 
 from dissect.target import Target
 from dissect.target.exceptions import UnsupportedPluginError
-from dissect.target.helpers.fsutil import open_decompress
 from dissect.target.helpers.record import DynamicDescriptor, TargetRecordDescriptor
 from dissect.target.helpers.utils import year_rollover_helper
 from dissect.target.plugin import Plugin, alias, export
+from dissect.target.plugins.os.unix.log.helpers import (
+    RE_LINE,
+    RE_TS,
+    is_iso_fmt,
+    iso_readlines,
+)
 
 log = logging.getLogger(__name__)
 
-RE_TS = re.compile(r"^[A-Za-z]{3}\s*\d{1,2}\s\d{1,2}:\d{2}:\d{2}")
-RE_TS_ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}")
-RE_LINE = re.compile(
-    r"""
-    \d{2}:\d{2}\s                           # First match on the similar ending of the different timestamps
-    (?P<hostname>\S+)\s                     # The hostname
-    (?P<service>\S+?)(\[(?P<pid>\d+)\])?:   # The service with optionally the PID between brackets
-    \s*(?P<message>.+?)\s*$                 # The log message stripped from spaces left and right
-    """,
-    re.VERBOSE,
-)
 
 # Generic regular expressions
 RE_IPV4_ADDRESS = re.compile(
@@ -347,27 +340,3 @@ def authlog(self) -> Iterator[Any]:
 
             for ts, line in iterable:
                 yield self._auth_log_builder.build_record(ts, auth_file, line)
-
-
-def iso_readlines(file: Path) -> Iterator[tuple[datetime, str]]:
-    """Iterator reading the provided auth log file in ISO format. Mimics ``year_rollover_helper`` behaviour."""
-    with open_decompress(file, "rt") as fh:
-        for line in fh:
-            if not (match := RE_TS_ISO.match(line)):
-                log.warning("No timestamp found in one of the lines in %s!", file)
-                log.debug("Skipping line: %s", line)
-                continue
-
-            try:
-                ts = datetime.strptime(match[0], "%Y-%m-%dT%H:%M:%S.%f%z")
-            except ValueError as e:
-                log.warning("Unable to parse ISO timestamp in line: %s", line)
-                log.debug("", exc_info=e)
-                continue
-
-            yield ts, line
-
-
-def is_iso_fmt(file: Path) -> bool:
-    """Determine if the provided auth log file uses new ISO format logging or not."""
-    return any(itertools.islice(iso_readlines(file), 0, 2))

dissect/target/plugins/os/unix/log/helpers.py

@@ -0,0 +1,46 @@
+import itertools
+import logging
+import re
+from datetime import datetime
+from pathlib import Path
+from typing import Iterator
+
+from dissect.target.helpers.fsutil import open_decompress
+
+log = logging.getLogger(__name__)
+
+RE_TS = re.compile(r"^[A-Za-z]{3}\s*\d{1,2}\s\d{1,2}:\d{2}:\d{2}")
+RE_TS_ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}")
+RE_LINE = re.compile(
+    r"""
+    \d{2}:\d{2}\s                           # First match on the similar ending of the different timestamps
+    (?:\S+)\s                               # The hostname, but do not capture it
+    (?P<service>\S+?)(\[(?P<pid>\d+)\])?:    # The service / daemon with optionally the PID between brackets
+    \s*(?P<message>.+?)\s*$                 # The log message stripped from spaces left and right
+    """,
+    re.VERBOSE,
+)
+
+
+def iso_readlines(file: Path) -> Iterator[tuple[datetime, str]]:
+    """Iterator reading the provided log file in ISO format. Mimics ``year_rollover_helper`` behaviour."""
+    with open_decompress(file, "rt") as fh:
+        for line in fh:
+            if not (match := RE_TS_ISO.match(line)):
+                log.warning("No timestamp found in one of the lines in %s!", file)
+                log.debug("Skipping line: %s", line)
+                continue
+
+            try:
+                ts = datetime.strptime(match[0], "%Y-%m-%dT%H:%M:%S.%f%z")
+            except ValueError as e:
+                log.warning("Unable to parse ISO timestamp in line: %s", line)
+                log.debug("", exc_info=e)
+                continue
+
+            yield ts, line
+
+
+def is_iso_fmt(file: Path) -> bool:
+    """Determine if the provided log file uses ISO 8601 timestamp format logging or not."""
+    return any(itertools.islice(iso_readlines(file), 0, 2))

dissect/target/plugins/os/unix/log/messages.py

@@ -11,25 +11,27 @@
 from dissect.target.helpers.record import TargetRecordDescriptor
 from dissect.target.helpers.utils import year_rollover_helper
 from dissect.target.plugin import Plugin, alias, export
+from dissect.target.plugins.os.unix.log.helpers import (
+    RE_LINE,
+    RE_TS,
+    is_iso_fmt,
+    iso_readlines,
+)
 
 MessagesRecord = TargetRecordDescriptor(
     "linux/log/messages",
     [
         ("datetime", "ts"),
-        ("string", "daemon"),
+        ("string", "service"),
         ("varint", "pid"),
         ("string", "message"),
         ("path", "source"),
     ],
 )
 
 DEFAULT_TS_LOG_FORMAT = "%b %d %H:%M:%S"
-RE_TS = re.compile(r"(\w+\s{1,2}\d+\s\d{2}:\d{2}:\d{2})")
-RE_DAEMON = re.compile(r"^[^:]+:\d+:\d+[^\[\]:]+\s([^\[:]+)[\[|:]{1}")
-RE_PID = re.compile(r"\w\[(\d+)\]")
-RE_MSG = re.compile(r"[^:]+:\d+:\d+[^:]+:\s(.*)$")
 RE_CLOUD_INIT_LINE = re.compile(
-    r"^(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) - (?P<daemon>.*)\[(?P<log_level>\w+)\]\: (?P<message>.*)$"
+    r"^(?P<ts>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) - (?P<service>.*)\[(?P<log_level>\w+)\]\: (?P<message>.*)$"
 )
 
 
@@ -56,7 +58,7 @@ def check_compatible(self) -> None:
     def messages(self) -> Iterator[MessagesRecord]:
         """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs.
 
-        Due to year rollover detection, the contents of the files are returned in reverse.
+        Due to year rollover detection, the log contents could be returned in reversed or mixed chronological order.
 
         The messages log file holds information about a variety of events such as the system error messages, system
         startups and shutdowns, change in the network configuration, etc. Aims to store valuable, non-debug and
@@ -75,16 +77,23 @@ def messages(self) -> Iterator[MessagesRecord]:
                 yield from self._parse_cloud_init_log(log_file, tzinfo)
                 continue
 
-            for ts, line in year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo):
-                daemon = dict(enumerate(RE_DAEMON.findall(line))).get(0)
-                pid = dict(enumerate(RE_PID.findall(line))).get(0)
-                message = dict(enumerate(RE_MSG.findall(line))).get(0, line)
+            if is_iso_fmt(log_file):
+                iterable = iso_readlines(log_file)
+
+            else:
+                iterable = year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo)
+
+            for ts, line in iterable:
+                match = RE_LINE.search(line)
+
+                if not match:
+                    self.target.log.warning("Unable to parse message line in %s", log_file)
+                    self.target.log.debug("Line %s", line)
+                    continue
 
                 yield MessagesRecord(
                     ts=ts,
-                    daemon=daemon,
-                    pid=pid,
-                    message=message,
+                    **match.groupdict(),
                     source=log_file,
                     _target=self.target,
                 )
@@ -134,7 +143,7 @@ def _parse_cloud_init_log(self, log_file: Path, tzinfo: tzinfo | None = timezone
 
                 yield MessagesRecord(
                     ts=ts,
-                    daemon=values["daemon"],
+                    service=values["service"],
                     pid=None,
                     message=values["message"],
                     source=log_file,