From f0f0aea212f719aa417f01cdc9ed71f3754bd16c Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 27 Nov 2024 03:47:58 -0500 Subject: [PATCH 1/7] Allow RegistryValueType to take any value (#956) --- dissect/target/helpers/regutil.py | 39 ++++++++++++++++++++++--------- tests/helpers/test_regutil.py | 32 ++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/dissect/target/helpers/regutil.py b/dissect/target/helpers/regutil.py index f55fcb8cc..26783fcd7 100644 --- a/dissect/target/helpers/regutil.py +++ b/dissect/target/helpers/regutil.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import BinaryIO, Iterator, Optional, TextIO, Union -from dissect.regf import regf +from dissect.regf import c_regf, regf from dissect.target.exceptions import ( RegistryError, @@ -31,16 +31,33 @@ class RegistryValueType(IntEnum): - NONE = regf.REG_NONE - SZ = regf.REG_SZ - EXPAND_SZ = regf.REG_EXPAND_SZ - BINARY = regf.REG_BINARY - DWORD = regf.REG_DWORD - DWORD_BIG_ENDIAN = regf.REG_DWORD_BIG_ENDIAN - MULTI_SZ = regf.REG_MULTI_SZ - FULL_RESOURCE_DESCRIPTOR = regf.REG_FULL_RESOURCE_DESCRIPTOR - RESOURCE_REQUIREMENTS_LIST = regf.REG_RESOURCE_REQUIREMENTS_LIST - QWORD = regf.REG_QWORD + """Registry value types as defined in ``winnt.h``. + + Resources: + - https://learn.microsoft.com/en-us/windows/win32/sysinfo/registry-value-types + - https://github.com/fox-it/dissect.regf/blob/main/dissect/regf/c_regf.py + """ + + NONE = c_regf.REG_NONE + SZ = c_regf.REG_SZ + EXPAND_SZ = c_regf.REG_EXPAND_SZ + BINARY = c_regf.REG_BINARY + DWORD = c_regf.REG_DWORD + DWORD_BIG_ENDIAN = c_regf.REG_DWORD_BIG_ENDIAN + LINK = c_regf.REG_LINK + MULTI_SZ = c_regf.REG_MULTI_SZ + RESOURCE_LIST = c_regf.REG_RESOURCE_LIST + FULL_RESOURCE_DESCRIPTOR = c_regf.REG_FULL_RESOURCE_DESCRIPTOR + RESOURCE_REQUIREMENTS_LIST = c_regf.REG_RESOURCE_REQUIREMENTS_LIST + QWORD = c_regf.REG_QWORD + + @classmethod + def _missing_(cls, value: int) -> IntEnum: + # Allow values other than defined members + member = int.__new__(cls, value) + member._name_ = None + member._value_ = value + return member class RegistryHive: diff --git a/tests/helpers/test_regutil.py b/tests/helpers/test_regutil.py index 0afed3c58..66aa7ecd0 100644 --- a/tests/helpers/test_regutil.py +++ b/tests/helpers/test_regutil.py @@ -1,6 +1,7 @@ -from typing import Union +from __future__ import annotations import pytest +from dissect.regf import c_regf from dissect.target.helpers.regutil import ( HiveCollection, @@ -172,7 +173,7 @@ def key_collection(hivecollection: HiveCollection) -> KeyCollection: ("some\\other\\bla\\", "bla"), ], ) -def test_registry_key_get(hive: RegistryHive, key_path: str, key_name: Union[str, RegistryKeyNotFoundError]) -> None: +def test_registry_key_get(hive: RegistryHive, key_path: str, key_name: str | RegistryKeyNotFoundError) -> None: key = hive.key("\\") if key_name is RegistryKeyNotFoundError: @@ -194,7 +195,7 @@ def test_registry_key_get(hive: RegistryHive, key_path: str, key_name: Union[str def test_key_collection_get( key_collection: KeyCollection, key_path: str, - key_name: Union[str, RegistryKeyNotFoundError], + key_name: str | RegistryKeyNotFoundError, ) -> None: if key_name is RegistryKeyNotFoundError: with pytest.raises(key_name): @@ -339,3 +340,28 @@ def test_glob_ext(key_collection: KeyCollection, pattern: str, key_paths: list[s collection_paths.append(key_collection.path) assert sorted(collection_paths) == sorted(key_paths) + + +@pytest.mark.parametrize( + "input, expected_name, expected_value", + [ + (c_regf.REG_NONE, "NONE", 0), + (c_regf.REG_SZ, "SZ", 1), + (c_regf.REG_EXPAND_SZ, "EXPAND_SZ", 2), + (c_regf.REG_BINARY, "BINARY", 3), + (c_regf.REG_DWORD, "DWORD", 4), + (c_regf.REG_DWORD_BIG_ENDIAN, "DWORD_BIG_ENDIAN", 5), + (c_regf.REG_LINK, "LINK", 6), + (c_regf.REG_MULTI_SZ, "MULTI_SZ", 7), + (c_regf.REG_RESOURCE_LIST, "RESOURCE_LIST", 8), + (c_regf.REG_FULL_RESOURCE_DESCRIPTOR, "FULL_RESOURCE_DESCRIPTOR", 9), + (c_regf.REG_RESOURCE_REQUIREMENTS_LIST, "RESOURCE_REQUIREMENTS_LIST", 10), + (c_regf.REG_QWORD, "QWORD", 11), + (1337, None, 1337), + ], +) +def test_registry_value_type_enum(input: int, expected_name: str | None, expected_value: int) -> None: + """test if registry value types are not parsed strictly within the Enum""" + regf_value = RegistryValueType(input) + assert regf_value == expected_value + assert regf_value.name == expected_name From 5c9eedf401409a21c972d4c50d066b3b99701403 Mon Sep 17 00:00:00 2001 From: wbi-ocd Date: Wed, 27 Nov 2024 11:22:33 +0100 Subject: [PATCH 2/7] Fix error in windows network plugins (#955) --- dissect/target/plugins/os/windows/network.py | 11 +++++++---- tests/plugins/os/windows/test_network.py | 8 ++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dissect/target/plugins/os/windows/network.py b/dissect/target/plugins/os/windows/network.py index d7c39ede8..8e268c017 100644 --- a/dissect/target/plugins/os/windows/network.py +++ b/dissect/target/plugins/os/windows/network.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from enum import IntEnum from functools import lru_cache from typing import Iterator @@ -224,11 +225,13 @@ def _try_value(subkey: RegistryKey, value: str) -> str | list | None: return None -def _get_config_value(key: RegistryKey, name: str) -> set: +def _get_config_value(key: RegistryKey, name: str, sep: str | None = None) -> set: value = _try_value(key, name) if not value or value in ("", "0.0.0.0", None, [], ["0.0.0.0"]): return set() - + if sep and isinstance(value, str): + re_sep = "|".join(map(re.escape, sep)) + value = re.split(re_sep, value) if isinstance(value, list): return set(value) @@ -355,11 +358,11 @@ def _extract_network_device_config(self, interface_id: str) -> list[dict[str, se dhcp_config["ip"].update(_get_config_value(key, "DhcpIPAddress")) dhcp_config["subnetmask"].update(_get_config_value(key, "DhcpSubnetMask")) dhcp_config["search_domain"].update(_get_config_value(key, "DhcpDomain")) - dhcp_config["dns"].update(_get_config_value(key, "DhcpNameServer")) + dhcp_config["dns"].update(_get_config_value(key, "DhcpNameServer", " ,")) # Extract static configuration from the registry static_config["gateway"].update(_get_config_value(key, "DefaultGateway")) - static_config["dns"].update(_get_config_value(key, "NameServer")) + static_config["dns"].update(_get_config_value(key, "NameServer", " ,")) static_config["search_domain"].update(_get_config_value(key, "Domain")) static_config["ip"].update(_get_config_value(key, "IPAddress")) static_config["subnetmask"].update(_get_config_value(key, "SubnetMask")) diff --git a/tests/plugins/os/windows/test_network.py b/tests/plugins/os/windows/test_network.py index a9237fcdf..5a7898dec 100644 --- a/tests/plugins/os/windows/test_network.py +++ b/tests/plugins/os/windows/test_network.py @@ -257,7 +257,7 @@ def test_windows_network_none( "DhcpIPAddress": "192.168.0.10", "IPAddress": "10.0.0.10", "DhcpNameServer": "192.168.0.2", - "NameServer": "10.0.0.2", + "NameServer": "10.0.0.2 10.0.0.3", "SubnetMask": "255.255.255.0", "DhcpSubnetMask": "255.255.255.0", "VlanID": 10, @@ -285,7 +285,7 @@ def test_windows_network_none( }, { "ip": ["10.0.0.10"], - "dns": ["10.0.0.2"], + "dns": ["10.0.0.2", "10.0.0.3"], "gateway": ["10.0.0.1"], "mac": ["FE:EE:EE:EE:EE:ED"], "subnetmask": ["255.255.255.0"], @@ -346,8 +346,8 @@ def test_network_dhcp_and_static( gateways.update(interface.gateway) macs.update(interface.mac) - assert interface.ip == expected["ip"] - assert interface.dns == expected["dns"] + assert sorted(map(str, interface.ip)) == expected["ip"] + assert sorted(map(str, interface.dns)) == expected["dns"] assert interface.gateway == expected["gateway"] assert interface.mac == expected["mac"] assert interface.network == expected["network"] From af1abe4d70a87f04ac00dcb0fcba1871c47808d7 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:28:02 -0500 Subject: [PATCH 3/7] Improve SQLite table exist checks (#958) --- dissect/target/loaders/itunes.py | 6 +- .../target/plugins/apps/browser/iexplore.py | 10 ++- dissect/target/plugins/os/unix/esxi/_os.py | 66 ++++++++-------- .../plugins/os/windows/activitiescache.py | 62 ++++++++------- dissect/target/plugins/os/windows/catroot.py | 9 ++- .../plugins/os/windows/notifications.py | 78 ++++++++++--------- 6 files changed, 123 insertions(+), 108 deletions(-) diff --git a/dissect/target/loaders/itunes.py b/dissect/target/loaders/itunes.py index 7a96a4630..52e600dce 100644 --- a/dissect/target/loaders/itunes.py +++ b/dissect/target/loaders/itunes.py @@ -163,8 +163,10 @@ def derive_key(self, password: str) -> bytes: def files(self) -> Iterator[FileInfo]: """Iterate all the files in this backup.""" - for row in self.manifest_db.table("Files").rows(): - yield FileInfo(self, row.fileID, row.domain, row.relativePath, row.flags, row.file) + + if table := self.manifest_db.table("Files"): + for row in table.rows(): + yield FileInfo(self, row.fileID, row.domain, row.relativePath, row.flags, row.file) class FileInfo: diff --git a/dissect/target/plugins/apps/browser/iexplore.py b/dissect/target/plugins/apps/browser/iexplore.py index 352614850..8806524ea 100644 --- a/dissect/target/plugins/apps/browser/iexplore.py +++ b/dissect/target/plugins/apps/browser/iexplore.py @@ -36,14 +36,18 @@ def find_containers(self, name: str) -> Iterator[table.Table]: All ``ContainerId`` values for the requested container name. """ try: - for container_record in self.db.table("Containers").records(): + table = self.db.table("Containers") + + for container_record in table.records(): if record_name := container_record.get("Name"): record_name = record_name.rstrip("\00").lower() if record_name == name.lower(): container_id = container_record.get("ContainerId") yield self.db.table(f"Container_{container_id}") - except KeyError: - pass + + except KeyError as e: + self.target.log.warning("Exception while parsing EseDB Containers table") + self.target.log.debug("", exc_info=e) def _iter_records(self, name: str) -> Iterator[record.Record]: """Yield records from a Webcache container. diff --git a/dissect/target/plugins/os/unix/esxi/_os.py b/dissect/target/plugins/os/unix/esxi/_os.py index c86a48a74..fcacdb9a3 100644 --- a/dissect/target/plugins/os/unix/esxi/_os.py +++ b/dissect/target/plugins/os/unix/esxi/_os.py @@ -472,37 +472,39 @@ def parse_config_store(fh: BinaryIO) -> dict[str, Any]: db = sqlite3.SQLite3(fh) store = {} - for row in db.table("Config").rows(): - component_name = row.Component - config_group_name = row.ConfigGroup - value_group_name = row.Name - identifier_name = row.Identifier - - if component_name not in store: - store[component_name] = {} - component = store[component_name] - - if config_group_name not in component: - component[config_group_name] = {} - config_group = component[config_group_name] - - if value_group_name not in config_group: - config_group[value_group_name] = {} - value_group = config_group[value_group_name] - - if identifier_name not in value_group: - value_group[identifier_name] = {} - identifier = value_group[identifier_name] - - identifier["modified_time"] = row.ModifiedTime - identifier["creation_time"] = row.CreationTime - identifier["version"] = row.Version - identifier["success"] = row.Success - identifier["auto_conf_value"] = json.loads(row.AutoConfValue) if row.AutoConfValue else None - identifier["user_value"] = json.loads(row.UserValue) if row.UserValue else None - identifier["vital_value"] = json.loads(row.VitalValue) if row.VitalValue else None - identifier["cached_value"] = json.loads(row.CachedValue) if row.CachedValue else None - identifier["desired_value"] = json.loads(row.DesiredValue) if row.DesiredValue else None - identifier["revision"] = row.Revision + + if table := db.table("Config"): + for row in table.rows(): + component_name = row.Component + config_group_name = row.ConfigGroup + value_group_name = row.Name + identifier_name = row.Identifier + + if component_name not in store: + store[component_name] = {} + component = store[component_name] + + if config_group_name not in component: + component[config_group_name] = {} + config_group = component[config_group_name] + + if value_group_name not in config_group: + config_group[value_group_name] = {} + value_group = config_group[value_group_name] + + if identifier_name not in value_group: + value_group[identifier_name] = {} + identifier = value_group[identifier_name] + + identifier["modified_time"] = row.ModifiedTime + identifier["creation_time"] = row.CreationTime + identifier["version"] = row.Version + identifier["success"] = row.Success + identifier["auto_conf_value"] = json.loads(row.AutoConfValue) if row.AutoConfValue else None + identifier["user_value"] = json.loads(row.UserValue) if row.UserValue else None + identifier["vital_value"] = json.loads(row.VitalValue) if row.VitalValue else None + identifier["cached_value"] = json.loads(row.CachedValue) if row.CachedValue else None + identifier["desired_value"] = json.loads(row.DesiredValue) if row.DesiredValue else None + identifier["revision"] = row.Revision return store diff --git a/dissect/target/plugins/os/windows/activitiescache.py b/dissect/target/plugins/os/windows/activitiescache.py index fcf106cf6..c945f423a 100644 --- a/dissect/target/plugins/os/windows/activitiescache.py +++ b/dissect/target/plugins/os/windows/activitiescache.py @@ -116,36 +116,38 @@ def activitiescache(self) -> Iterator[ActivitiesCacheRecord]: for user, cache_file in self.cachefiles: fh = cache_file.open() db = sqlite3.SQLite3(fh) - for r in db.table("Activity").rows(): - yield ActivitiesCacheRecord( - start_time=mkts(r["[StartTime]"]), - end_time=mkts(r["[EndTime]"]), - last_modified_time=mkts(r["[LastModifiedTime]"]), - last_modified_on_client=mkts(r["[LastModifiedOnClient]"]), - original_last_modified_on_client=mkts(r["[OriginalLastModifiedOnClient]"]), - expiration_time=mkts(r["[ExpirationTime]"]), - app_id=r["[AppId]"], - enterprise_id=r["[EnterpriseId]"], - app_activity_id=r["[AppActivityId]"], - group_app_activity_id=r["[GroupAppActivityId]"], - group=r["[Group]"], - activity_type=r["[ActivityType]"], - activity_status=r["[ActivityStatus]"], - priority=r["[Priority]"], - match_id=r["[MatchId]"], - etag=r["[ETag]"], - tag=r["[Tag]"], - is_local_only=r["[IsLocalOnly]"], - created_in_cloud=r["[CreatedInCloud]"], - platform_device_id=r["[PlatformDeviceId]"], - package_id_hash=r["[PackageIdHash]"], - id=r["[Id]"], - payload=r["[Payload]"], - original_payload=r["[OriginalPayload]"], - clipboard_payload=r["[ClipboardPayload]"], - _target=self.target, - _user=user, - ) + + if table := db.table("Activity"): + for r in table.rows(): + yield ActivitiesCacheRecord( + start_time=mkts(r["[StartTime]"]), + end_time=mkts(r["[EndTime]"]), + last_modified_time=mkts(r["[LastModifiedTime]"]), + last_modified_on_client=mkts(r["[LastModifiedOnClient]"]), + original_last_modified_on_client=mkts(r["[OriginalLastModifiedOnClient]"]), + expiration_time=mkts(r["[ExpirationTime]"]), + app_id=r["[AppId]"], + enterprise_id=r["[EnterpriseId]"], + app_activity_id=r["[AppActivityId]"], + group_app_activity_id=r["[GroupAppActivityId]"], + group=r["[Group]"], + activity_type=r["[ActivityType]"], + activity_status=r["[ActivityStatus]"], + priority=r["[Priority]"], + match_id=r["[MatchId]"], + etag=r["[ETag]"], + tag=r["[Tag]"], + is_local_only=r["[IsLocalOnly]"], + created_in_cloud=r["[CreatedInCloud]"], + platform_device_id=r["[PlatformDeviceId]"], + package_id_hash=r["[PackageIdHash]"], + id=r["[Id]"], + payload=r["[Payload]"], + original_payload=r["[OriginalPayload]"], + clipboard_payload=r["[ClipboardPayload]"], + _target=self.target, + _user=user, + ) def mkts(ts: int) -> datetime | None: diff --git a/dissect/target/plugins/os/windows/catroot.py b/dissect/target/plugins/os/windows/catroot.py index 9fc38df38..6de792100 100644 --- a/dissect/target/plugins/os/windows/catroot.py +++ b/dissect/target/plugins/os/windows/catroot.py @@ -217,12 +217,15 @@ def catdb(self) -> Iterator[CatrootRecord]: with ese_file.open("rb") as fh: ese_db = EseDB(fh) - tables = [table.name for table in ese_db.tables()] for hash_type, table_name in [("sha256", "HashCatNameTableSHA256"), ("sha1", "HashCatNameTableSHA1")]: - if table_name not in tables: + try: + table = ese_db.table(table_name) + except KeyError as e: + self.target.log.warning("EseDB %s has no table %s", ese_file, table_name) + self.target.log.debug("", exc_info=e) continue - for record in ese_db.table(table_name).records(): + for record in table.records(): file_digest = digest() setattr(file_digest, hash_type, record.get("HashCatNameTable_HashCol").hex()) catroot_names = record.get("HashCatNameTable_CatNameCol").decode().rstrip("|").split("|") diff --git a/dissect/target/plugins/os/windows/notifications.py b/dissect/target/plugins/os/windows/notifications.py index fa10526f5..27f2bdbc7 100644 --- a/dissect/target/plugins/os/windows/notifications.py +++ b/dissect/target/plugins/os/windows/notifications.py @@ -442,43 +442,45 @@ def wpndatabase(self) -> Iterator[WpnDatabaseNotificationRecord | WpnDatabaseNot """ for user, wpndatabase in self.wpndb_files: db = sqlite3.SQLite3(wpndatabase.open()) - handlers = {} - for row in db.table("NotificationHandler").rows(): - handlers[row["[RecordId]"]] = WpnDatabaseNotificationHandlerRecord( - created_time=datetime.datetime.strptime(row["[CreatedTime]"], "%Y-%m-%d %H:%M:%S"), - modified_time=datetime.datetime.strptime(row["[ModifiedTime]"], "%Y-%m-%d %H:%M:%S"), - id=row["[RecordId]"], - primary_id=row["[PrimaryId]"], - wns_id=row["[WNSId]"], - handler_type=row["[HandlerType]"], - wnf_event_name=row["[WNFEventName]"], - system_data_property_set=row["[SystemDataPropertySet]"], - _target=self.target, - _user=user, - ) - - for row in db.table("Notification").rows(): - record = WpnDatabaseNotificationRecord( - arrival_time=wintimestamp(row["[ArrivalTime]"]), - expiry_time=wintimestamp(row["[ExpiryTime]"]), - order=row["[Order]"], - id=row["[Id]"], - handler_id=row["[HandlerId]"], - activity_id=UUID(bytes=row["[ActivityId]"]), - type=row["[Type]"], - payload=row["[Payload]"], - payload_type=row["[PayloadType]"], - tag=row["[Tag]"], - group=row["[Group]"], - boot_id=row["[BootId]"], - expires_on_reboot=row["[ExpiresOnReboot]"] != "FALSE", - _target=self.target, - _user=user, - ) - handler = handlers.get(row["[HandlerId]"]) - if handler: - yield GroupedRecord("windows/notification/wpndatabase/grouped", [record, handler]) - else: - yield record + if table := db.table("NotificationHandler"): + for row in table.rows(): + handlers[row["[RecordId]"]] = WpnDatabaseNotificationHandlerRecord( + created_time=datetime.datetime.strptime(row["[CreatedTime]"], "%Y-%m-%d %H:%M:%S"), + modified_time=datetime.datetime.strptime(row["[ModifiedTime]"], "%Y-%m-%d %H:%M:%S"), + id=row["[RecordId]"], + primary_id=row["[PrimaryId]"], + wns_id=row["[WNSId]"], + handler_type=row["[HandlerType]"], + wnf_event_name=row["[WNFEventName]"], + system_data_property_set=row["[SystemDataPropertySet]"], + _target=self.target, + _user=user, + ) + + if table := db.table("Notification"): + for row in table.rows(): + record = WpnDatabaseNotificationRecord( + arrival_time=wintimestamp(row["[ArrivalTime]"]), + expiry_time=wintimestamp(row["[ExpiryTime]"]), + order=row["[Order]"], + id=row["[Id]"], + handler_id=row["[HandlerId]"], + activity_id=UUID(bytes=row["[ActivityId]"]), + type=row["[Type]"], + payload=row["[Payload]"], + payload_type=row["[PayloadType]"], + tag=row["[Tag]"], + group=row["[Group]"], + boot_id=row["[BootId]"], + expires_on_reboot=row["[ExpiresOnReboot]"] != "FALSE", + _target=self.target, + _user=user, + ) + handler = handlers.get(row["[HandlerId]"]) + + if handler: + yield GroupedRecord("windows/notification/wpndatabase/grouped", [record, handler]) + else: + yield record From f36ce5976d7665fa25c6259fec58ded9a0f2000d Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:50:09 +0100 Subject: [PATCH 4/7] Add support for ISO 8601 timestamps in syslogs (#907) Co-authored-by: Paul M <22234727+Poeloe@users.noreply.github.com> --- .../plugins/os/unix/linux/network_managers.py | 2 +- dissect/target/plugins/os/unix/log/auth.py | 43 +++-------------- dissect/target/plugins/os/unix/log/helpers.py | 46 +++++++++++++++++++ .../target/plugins/os/unix/log/messages.py | 39 ++++++++++------ tests/plugins/os/unix/log/test_messages.py | 31 ++++++++++++- 5 files changed, 106 insertions(+), 55 deletions(-) create mode 100644 dissect/target/plugins/os/unix/log/helpers.py diff --git a/dissect/target/plugins/os/unix/linux/network_managers.py b/dissect/target/plugins/os/unix/linux/network_managers.py index d3471769b..4fb4def76 100644 --- a/dissect/target/plugins/os/unix/linux/network_managers.py +++ b/dissect/target/plugins/os/unix/linux/network_managers.py @@ -567,7 +567,7 @@ def records_enumerate(iterable: Iterable) -> Iterator[tuple[int, JournalRecord | continue # Debian and CentOS dhclient - if hasattr(record, "daemon") and record.daemon == "dhclient" and "bound to" in line: + if hasattr(record, "service") and record.service == "dhclient" and "bound to" in line: ip = line.split("bound to")[1].split(" ")[1].strip() ips.add(ip) continue diff --git a/dissect/target/plugins/os/unix/log/auth.py b/dissect/target/plugins/os/unix/log/auth.py index 4be43260e..f7e37a804 100644 --- a/dissect/target/plugins/os/unix/log/auth.py +++ b/dissect/target/plugins/os/unix/log/auth.py @@ -1,6 +1,5 @@ from __future__ import annotations -import itertools import logging import re from abc import ABC, abstractmethod @@ -12,24 +11,18 @@ from dissect.target import Target from dissect.target.exceptions import UnsupportedPluginError -from dissect.target.helpers.fsutil import open_decompress from dissect.target.helpers.record import DynamicDescriptor, TargetRecordDescriptor from dissect.target.helpers.utils import year_rollover_helper from dissect.target.plugin import Plugin, alias, export +from dissect.target.plugins.os.unix.log.helpers import ( + RE_LINE, + RE_TS, + is_iso_fmt, + iso_readlines, +) log = logging.getLogger(__name__) -RE_TS = re.compile(r"^[A-Za-z]{3}\s*\d{1,2}\s\d{1,2}:\d{2}:\d{2}") -RE_TS_ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}") -RE_LINE = re.compile( - r""" - \d{2}:\d{2}\s # First match on the similar ending of the different timestamps - (?P\S+)\s # The hostname - (?P\S+?)(\[(?P\d+)\])?: # The service with optionally the PID between brackets - \s*(?P.+?)\s*$ # The log message stripped from spaces left and right - """, - re.VERBOSE, -) # Generic regular expressions RE_IPV4_ADDRESS = re.compile( @@ -347,27 +340,3 @@ def authlog(self) -> Iterator[Any]: for ts, line in iterable: yield self._auth_log_builder.build_record(ts, auth_file, line) - - -def iso_readlines(file: Path) -> Iterator[tuple[datetime, str]]: - """Iterator reading the provided auth log file in ISO format. Mimics ``year_rollover_helper`` behaviour.""" - with open_decompress(file, "rt") as fh: - for line in fh: - if not (match := RE_TS_ISO.match(line)): - log.warning("No timestamp found in one of the lines in %s!", file) - log.debug("Skipping line: %s", line) - continue - - try: - ts = datetime.strptime(match[0], "%Y-%m-%dT%H:%M:%S.%f%z") - except ValueError as e: - log.warning("Unable to parse ISO timestamp in line: %s", line) - log.debug("", exc_info=e) - continue - - yield ts, line - - -def is_iso_fmt(file: Path) -> bool: - """Determine if the provided auth log file uses new ISO format logging or not.""" - return any(itertools.islice(iso_readlines(file), 0, 2)) diff --git a/dissect/target/plugins/os/unix/log/helpers.py b/dissect/target/plugins/os/unix/log/helpers.py new file mode 100644 index 000000000..f37c4d8c3 --- /dev/null +++ b/dissect/target/plugins/os/unix/log/helpers.py @@ -0,0 +1,46 @@ +import itertools +import logging +import re +from datetime import datetime +from pathlib import Path +from typing import Iterator + +from dissect.target.helpers.fsutil import open_decompress + +log = logging.getLogger(__name__) + +RE_TS = re.compile(r"^[A-Za-z]{3}\s*\d{1,2}\s\d{1,2}:\d{2}:\d{2}") +RE_TS_ISO = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}\+\d{2}:\d{2}") +RE_LINE = re.compile( + r""" + \d{2}:\d{2}\s # First match on the similar ending of the different timestamps + (?:\S+)\s # The hostname, but do not capture it + (?P\S+?)(\[(?P\d+)\])?: # The service / daemon with optionally the PID between brackets + \s*(?P.+?)\s*$ # The log message stripped from spaces left and right + """, + re.VERBOSE, +) + + +def iso_readlines(file: Path) -> Iterator[tuple[datetime, str]]: + """Iterator reading the provided log file in ISO format. Mimics ``year_rollover_helper`` behaviour.""" + with open_decompress(file, "rt") as fh: + for line in fh: + if not (match := RE_TS_ISO.match(line)): + log.warning("No timestamp found in one of the lines in %s!", file) + log.debug("Skipping line: %s", line) + continue + + try: + ts = datetime.strptime(match[0], "%Y-%m-%dT%H:%M:%S.%f%z") + except ValueError as e: + log.warning("Unable to parse ISO timestamp in line: %s", line) + log.debug("", exc_info=e) + continue + + yield ts, line + + +def is_iso_fmt(file: Path) -> bool: + """Determine if the provided log file uses ISO 8601 timestamp format logging or not.""" + return any(itertools.islice(iso_readlines(file), 0, 2)) diff --git a/dissect/target/plugins/os/unix/log/messages.py b/dissect/target/plugins/os/unix/log/messages.py index 37f80f504..43bfed43e 100644 --- a/dissect/target/plugins/os/unix/log/messages.py +++ b/dissect/target/plugins/os/unix/log/messages.py @@ -11,12 +11,18 @@ from dissect.target.helpers.record import TargetRecordDescriptor from dissect.target.helpers.utils import year_rollover_helper from dissect.target.plugin import Plugin, alias, export +from dissect.target.plugins.os.unix.log.helpers import ( + RE_LINE, + RE_TS, + is_iso_fmt, + iso_readlines, +) MessagesRecord = TargetRecordDescriptor( "linux/log/messages", [ ("datetime", "ts"), - ("string", "daemon"), + ("string", "service"), ("varint", "pid"), ("string", "message"), ("path", "source"), @@ -24,12 +30,8 @@ ) DEFAULT_TS_LOG_FORMAT = "%b %d %H:%M:%S" -RE_TS = re.compile(r"(\w+\s{1,2}\d+\s\d{2}:\d{2}:\d{2})") -RE_DAEMON = re.compile(r"^[^:]+:\d+:\d+[^\[\]:]+\s([^\[:]+)[\[|:]{1}") -RE_PID = re.compile(r"\w\[(\d+)\]") -RE_MSG = re.compile(r"[^:]+:\d+:\d+[^:]+:\s(.*)$") RE_CLOUD_INIT_LINE = re.compile( - r"^(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) - (?P.*)\[(?P\w+)\]\: (?P.*)$" + r"^(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}) - (?P.*)\[(?P\w+)\]\: (?P.*)$" ) @@ -56,7 +58,7 @@ def check_compatible(self) -> None: def messages(self) -> Iterator[MessagesRecord]: """Return contents of /var/log/messages*, /var/log/syslog* and cloud-init logs. - Due to year rollover detection, the contents of the files are returned in reverse. + Due to year rollover detection, the log contents could be returned in reversed or mixed chronological order. The messages log file holds information about a variety of events such as the system error messages, system startups and shutdowns, change in the network configuration, etc. Aims to store valuable, non-debug and @@ -75,16 +77,23 @@ def messages(self) -> Iterator[MessagesRecord]: yield from self._parse_cloud_init_log(log_file, tzinfo) continue - for ts, line in year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo): - daemon = dict(enumerate(RE_DAEMON.findall(line))).get(0) - pid = dict(enumerate(RE_PID.findall(line))).get(0) - message = dict(enumerate(RE_MSG.findall(line))).get(0, line) + if is_iso_fmt(log_file): + iterable = iso_readlines(log_file) + + else: + iterable = year_rollover_helper(log_file, RE_TS, DEFAULT_TS_LOG_FORMAT, tzinfo) + + for ts, line in iterable: + match = RE_LINE.search(line) + + if not match: + self.target.log.warning("Unable to parse message line in %s", log_file) + self.target.log.debug("Line %s", line) + continue yield MessagesRecord( ts=ts, - daemon=daemon, - pid=pid, - message=message, + **match.groupdict(), source=log_file, _target=self.target, ) @@ -134,7 +143,7 @@ def _parse_cloud_init_log(self, log_file: Path, tzinfo: tzinfo | None = timezone yield MessagesRecord( ts=ts, - daemon=values["daemon"], + service=values["service"], pid=None, message=values["message"], source=log_file, diff --git a/tests/plugins/os/unix/log/test_messages.py b/tests/plugins/os/unix/log/test_messages.py index 08c297839..33a259cfb 100644 --- a/tests/plugins/os/unix/log/test_messages.py +++ b/tests/plugins/os/unix/log/test_messages.py @@ -10,6 +10,7 @@ from dissect.target.filesystem import VirtualFilesystem from dissect.target.filesystems.tar import TarFilesystem from dissect.target.plugins.general import default +from dissect.target.plugins.os.unix._os import UnixPlugin from dissect.target.plugins.os.unix.log.messages import MessagesPlugin, MessagesRecord from tests._utils import absolute_path @@ -125,16 +126,42 @@ def test_unix_messages_cloud_init(target_unix: Target, fs_unix: VirtualFilesyste assert len(results) == 4 assert results[0].ts == datetime(2005, 8, 9, 11, 55, 21, 0, tzinfo=ZoneInfo("Europe/Amsterdam")) - assert results[0].daemon == "foo.py" + assert results[0].service == "foo.py" assert results[0].pid is None assert results[0].message == "This is a cloud-init message!" assert results[0].source == "/var/log/installer/cloud-init.log" assert results[-1].ts == datetime(2005, 8, 9, 11, 55, 21, 1_000, tzinfo=ZoneInfo("Europe/Amsterdam")) - assert results[-1].daemon == "util.py" + assert results[-1].service == "util.py" assert results[-1].pid is None assert ( results[-1].message == "Cloud-init v. 1.2.3-4ubuntu5 running 'init-local' at Tue, 9 Aug 2005 11:55:21 +0000. Up 13.37 seconds." # noqa: E501 ) assert results[-1].source == "/var/log/installer/cloud-init.log.1.gz" + + +def test_unix_messages_ts_iso_8601_format(target_unix: Target, fs_unix: VirtualFilesystem) -> None: + """test if we correctly detect and parse ISO 8601 formatted syslog logs.""" + + fs_unix.map_file_fh("/etc/hostname", BytesIO(b"hostname")) + messages = """ + 2024-12-31T13:37:00.123456+02:00 hostname systemd[1]: Started anacron.service - Run anacron jobs. + 2024-12-31T13:37:00.123456+02:00 hostname anacron[1337]: Anacron 2.3 started on 2024-12-31 + 2024-12-31T13:37:00.123456+02:00 hostname anacron[1337]: Normal exit (0 jobs run) + 2024-12-31T13:37:00.123456+02:00 hostname systemd[1]: anacron.service: Deactivated successfully. + """ + fs_unix.map_file_fh("/var/log/syslog.1", BytesIO(gzip.compress(textwrap.dedent(messages).encode()))) + + target_unix.add_plugin(UnixPlugin) + target_unix.add_plugin(MessagesPlugin) + results = sorted(list(target_unix.syslog()), key=lambda r: r.ts) + + assert len(results) == 4 + + assert results[0].hostname == "hostname" + assert results[0].service == "systemd" + assert results[0].pid == 1 + assert results[0].ts == datetime(2024, 12, 31, 11, 37, 0, 123456, tzinfo=timezone.utc) + assert results[0].message == "Started anacron.service - Run anacron jobs." + assert results[0].source == "/var/log/syslog.1" From 4388f767910edd1317e6e98e818db5be145f224e Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 3 Dec 2024 16:50:17 -0500 Subject: [PATCH 5/7] Improve cit plugin exception handling (#961) --- dissect/target/plugins/os/windows/regf/cit.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/dissect/target/plugins/os/windows/regf/cit.py b/dissect/target/plugins/os/windows/regf/cit.py index 4d3745e90..ff44c5d8a 100644 --- a/dissect/target/plugins/os/windows/regf/cit.py +++ b/dissect/target/plugins/os/windows/regf/cit.py @@ -632,8 +632,8 @@ def local_wintimestamp(target, ts): class CITPlugin(Plugin): """Plugin that parses CIT data from the registry. - Reference: - - https://dfir.ru/2018/12/02/the-cit-database-and-the-syscache-hive/ + References: + - https://dfir.ru/2018/12/02/the-cit-database-and-the-syscache-hive/ """ __namespace__ = "cit" @@ -641,7 +641,7 @@ class CITPlugin(Plugin): KEY = "HKLM\\Software\\Microsoft\\Windows NT\\CurrentVersion\\AppCompatFlags\\CIT" def check_compatible(self) -> None: - if not len(list(self.target.registry.keys(self.KEY))) > 0: + if not list(self.target.registry.keys(self.KEY)): raise UnsupportedPluginError("No CIT registry key found") @export(record=get_args(CITRecords)) @@ -770,8 +770,9 @@ def cit(self) -> Iterator[CITRecords]: yield from _yield_bitmap_records( self.target, cit, entry.use_data.bitmaps.foreground, CITProgramBitmapForegroundRecord ) - except Exception: - self.target.log.exception("Failed to parse CIT value: %s", value.name) + except Exception as e: + self.target.log.warning("Failed to parse CIT value: %s", value.name) + self.target.log.debug("", exc_info=e) @export(record=CITPostUpdateUseInfoRecord) def puu(self) -> Iterator[CITPostUpdateUseInfoRecord]: @@ -788,10 +789,16 @@ def puu(self) -> Iterator[CITPostUpdateUseInfoRecord]: for reg_key in keys: for key in self.target.registry.keys(reg_key): try: - puu = c_cit.CIT_POST_UPDATE_USE_INFO(key.value("PUUActive").value) + key_value = key.value("PUUActive").value + puu = c_cit.CIT_POST_UPDATE_USE_INFO(key_value) except RegistryValueNotFoundError: continue + except EOFError as e: + self.target.log.warning("Exception reading CIT structure in key %s", key.path) + self.target.log.debug("Unable to parse value %s", key_value, exc_info=e) + continue + yield CITPostUpdateUseInfoRecord( log_time_start=wintimestamp(puu.LogTimeStart), update_key=puu.UpdateKey, @@ -852,10 +859,16 @@ def dp(self) -> Iterator[CITDPRecord | CITDPDurationRecord]: for reg_key in keys: for key in self.target.registry.keys(reg_key): try: - dp = c_cit.CIT_DP_DATA(key.value("DP").value) + key_value = key.value("DP").value + dp = c_cit.CIT_DP_DATA(key_value) except RegistryValueNotFoundError: continue + except EOFError as e: + self.target.log.warning("Exception reading CIT structure in key %s", key.path) + self.target.log.debug("Unable to parse value %s", key_value, exc_info=e) + continue + user = self.target.registry.get_user(key) log_time_start = wintimestamp(dp.LogTimeStart) From b681e755f45a4ab920db1d9e102db8b2d16e3258 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 4 Dec 2024 03:04:02 -0500 Subject: [PATCH 6/7] Improve catroot plugin exception handling (#962) --- dissect/target/plugins/os/windows/catroot.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dissect/target/plugins/os/windows/catroot.py b/dissect/target/plugins/os/windows/catroot.py index 6de792100..b7d1c4e1f 100644 --- a/dissect/target/plugins/os/windows/catroot.py +++ b/dissect/target/plugins/os/windows/catroot.py @@ -227,8 +227,14 @@ def catdb(self) -> Iterator[CatrootRecord]: for record in table.records(): file_digest = digest() - setattr(file_digest, hash_type, record.get("HashCatNameTable_HashCol").hex()) - catroot_names = record.get("HashCatNameTable_CatNameCol").decode().rstrip("|").split("|") + + try: + setattr(file_digest, hash_type, record.get("HashCatNameTable_HashCol").hex()) + catroot_names = record.get("HashCatNameTable_CatNameCol").decode().rstrip("|").split("|") + except Exception as e: + self.target.log.warning("Unable to parse catroot names for %s in %s", record, ese_file) + self.target.log.debug("", exc_info=e) + continue for catroot_name in catroot_names: yield CatrootRecord( From e728a2ccb3a40850ee8d53e67e4d5d6a5b8a6ed6 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 4 Dec 2024 04:09:29 -0500 Subject: [PATCH 7/7] Improve lnk plugin exception handling (#963) --- dissect/target/plugins/os/windows/lnk.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/dissect/target/plugins/os/windows/lnk.py b/dissect/target/plugins/os/windows/lnk.py index eb65f6030..ceb0cb584 100644 --- a/dissect/target/plugins/os/windows/lnk.py +++ b/dissect/target/plugins/os/windows/lnk.py @@ -1,4 +1,6 @@ -from typing import Iterator, Optional +from __future__ import annotations + +from typing import Iterator from dissect.shellitem.lnk import Lnk from dissect.util import ts @@ -34,7 +36,7 @@ ) -def parse_lnk_file(target: Target, lnk_file: Lnk, lnk_path: TargetPath) -> Iterator[LnkRecord]: +def parse_lnk_file(target: Target, lnk_file: Lnk, lnk_path: TargetPath) -> LnkRecord: # we need to get the active codepage from the system to properly decode some values codepage = target.codepage or "ascii" @@ -132,7 +134,7 @@ def check_compatible(self) -> None: @arg("--path", "-p", dest="path", default=None, help="Path to directory or .lnk file in target") @export(record=LnkRecord) - def lnk(self, path: Optional[str] = None) -> Iterator[LnkRecord]: + def lnk(self, path: str | None = None) -> Iterator[LnkRecord]: """Parse all .lnk files in /ProgramData, /Users, and /Windows or from a specified path in record format. Yields a LnkRecord record with the following fields: @@ -160,10 +162,14 @@ def lnk(self, path: Optional[str] = None) -> Iterator[LnkRecord]: """ for entry in self.lnk_entries(path): - lnk_file = Lnk(entry.open()) - yield parse_lnk_file(self.target, lnk_file, entry) - - def lnk_entries(self, path: Optional[str] = None) -> Iterator[TargetPath]: + try: + lnk_file = Lnk(entry.open()) + yield parse_lnk_file(self.target, lnk_file, entry) + except Exception as e: + self.target.log.warning("Failed to parse link file %s", lnk_file) + self.target.log.debug("", exc_info=e) + + def lnk_entries(self, path: str | None = None) -> Iterator[TargetPath]: if path: target_path = self.target.fs.path(path) if not target_path.exists():