From 2e2a1206beab7c003cd0b5e7c4bc06b9ced68c99 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Sun, 5 May 2024 21:36:56 +0200 Subject: [PATCH 01/26] Log warning in case file entry can't be mapped This occurs with some Velociraptor collections. It is currently unknown why this issue occurs. --- dissect/target/filesystems/zip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dissect/target/filesystems/zip.py b/dissect/target/filesystems/zip.py index 6c4d33c9d..f6a4dc5b8 100644 --- a/dissect/target/filesystems/zip.py +++ b/dissect/target/filesystems/zip.py @@ -57,7 +57,10 @@ def __init__( entry_cls = ZipFilesystemDirectoryEntry if member.is_dir() else ZipFilesystemEntry file_entry = entry_cls(self, rel_name, member) - self._fs.map_file_entry(rel_name, file_entry) + try: + self._fs.map_file_entry(rel_name, file_entry) + except Exception: + log.warning("Failed to map %s into VFS", file_entry) @staticmethod def _detect(fh: BinaryIO) -> bool: From f13bd46762998430835862a2b964f60e6cff14d5 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Sun, 5 May 2024 23:11:49 +0200 Subject: [PATCH 02/26] Fix Velociraptor URL-encoding of FS entries The names of the files and directories collected by Velociraptor are URL-encoded, which are decoded by the ZIP loader in order to prevent errors when executing plugins. This mainly impacts Unix targets because files like `%2Ebash_history` -> `.bash_history` are used by the plugins. Loading a directory is removed, because using the ZIP as a target is the preferred and enforced method. --- dissect/target/filesystems/zip.py | 6 +++ dissect/target/loaders/dir.py | 12 ++++- dissect/target/loaders/velociraptor.py | 32 +++++------ tests/loaders/test_velociraptor.py | 74 ++++---------------------- 4 files changed, 39 insertions(+), 85 deletions(-) diff --git a/dissect/target/filesystems/zip.py b/dissect/target/filesystems/zip.py index f6a4dc5b8..b18fc3ca1 100644 --- a/dissect/target/filesystems/zip.py +++ b/dissect/target/filesystems/zip.py @@ -5,6 +5,7 @@ import zipfile from datetime import datetime, timezone from typing import BinaryIO, Optional +from urllib.parse import unquote from dissect.util.stream import BufferedStream @@ -34,6 +35,7 @@ def __init__( self, fh: BinaryIO, base: Optional[str] = None, + decode_name: bool = False, *args, **kwargs, ): @@ -57,6 +59,10 @@ def __init__( entry_cls = ZipFilesystemDirectoryEntry if member.is_dir() else ZipFilesystemEntry file_entry = entry_cls(self, rel_name, member) + + if decode_name: + rel_name = unquote(rel_name) + try: self._fs.map_file_entry(rel_name, file_entry) except Exception: diff --git a/dissect/target/loaders/dir.py b/dissect/target/loaders/dir.py index 22a55e549..497875235 100644 --- a/dissect/target/loaders/dir.py +++ b/dissect/target/loaders/dir.py @@ -36,7 +36,9 @@ def find_entry_path(path: Path) -> str | None: return prefix -def map_dirs(target: Target, dirs: list[Path | tuple[str, Path]], os_type: str, **kwargs) -> None: +def map_dirs( + target: Target, dirs: list[Path | tuple[str, Path]], os_type: str, decode_name: bool = False, **kwargs +) -> None: """Map directories as filesystems into the given target. Args: @@ -59,7 +61,13 @@ def map_dirs(target: Target, dirs: list[Path | tuple[str, Path]], os_type: str, drive_letter = path.name[0] if isinstance(path, zipfile.Path): - dfs = ZipFilesystem(path.root.fp, path.at, alt_separator=alt_separator, case_sensitive=case_sensitive) + dfs = ZipFilesystem( + path.root.fp, + path.at, + alt_separator=alt_separator, + case_sensitive=case_sensitive, + decode_name=decode_name, + ) else: dfs = DirectoryFilesystem(path, alt_separator=alt_separator, case_sensitive=case_sensitive) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 68737cf7f..41d82ad13 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -77,6 +77,9 @@ class VelociraptorLoader(DirLoader): Generic.Collectors.File (Windows) and Windows.KapeFiles.Targets (Windows) uses the accessors mft, ntfs, lazy_ntfs, ntfs_vss and auto. The loader supports a collection where multiple accessors were used. + The filesystem entries collected by Velociraptor are URL-encoded, which are decoded by the ZIP loader in order to + prevent errors when executing plugins. + References: - https://www.rapid7.com/products/velociraptor/ - https://docs.velociraptor.app/ @@ -87,13 +90,13 @@ def __init__(self, path: Path, **kwargs): super().__init__(path) if path.suffix == ".zip": - log.warning( - f"Velociraptor target {path!r} is compressed, which will slightly affect performance. " - "Consider uncompressing the archive and passing the uncompressed folder to Dissect." - ) self.root = zipfile.Path(path) - else: - self.root = path + compression_type = zipfile.ZipFile(str(path)).getinfo("uploads.json").compress_type + if compression_type > 0: + log.warning( + f"Velociraptor target {path!r} is compressed, which will slightly affect performance. " + "Consider uncompressing the archive and passing the uncompressed folder to Dissect." + ) @staticmethod def detect(path: Path) -> bool: @@ -108,22 +111,15 @@ def detect(path: Path) -> bool: if path.suffix == ".zip": # novermin path = zipfile.Path(path) - if path.joinpath(FILESYSTEMS_ROOT).exists() and path.joinpath("uploads.json").exists(): - _, dirs = find_fs_directories(path) - return bool(dirs) + if path.joinpath(FILESYSTEMS_ROOT).exists() and path.joinpath("uploads.json").exists(): + _, dirs = find_fs_directories(path) + return bool(dirs) return False def map(self, target: Target) -> None: os_type, dirs = find_fs_directories(self.root) if os_type == OperatingSystem.WINDOWS: - # Velociraptor doesn't have the correct filenames for the paths "$J" and "$Secure:$SDS" - map_dirs( - target, - dirs, - os_type, - usnjrnl_path="$Extend/$UsnJrnl%3A$J", - sds_path="$Secure%3A$SDS", - ) + map_dirs(target, dirs, os_type, decode_name=True) else: - map_dirs(target, dirs, os_type) + map_dirs(target, dirs, os_type, decode_name=True) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 3b1291e19..0c8c1d402 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -42,40 +42,6 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: return root -@pytest.mark.parametrize( - "sub_dir, other_dir", - [ - ("mft", "auto"), - ("ntfs", "auto"), - ("ntfs_vss", "auto"), - ("lazy_ntfs", "auto"), - ("auto", "ntfs"), - ], -) -def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_path: Path) -> None: - root = create_root(sub_dir, tmp_path) - root.joinpath(f"uploads/{other_dir}/C%3A").mkdir(parents=True, exist_ok=True) - root.joinpath(f"uploads/{other_dir}/C%3A/other.txt").write_text("my first file") - - assert VelociraptorLoader.detect(root) is True - - loader = VelociraptorLoader(root) - loader.map(target_bare) - target_bare.apply() - - assert "sysvol" in target_bare.fs.mounts - assert "c:" in target_bare.fs.mounts - - usnjrnl_records = 0 - for fs in target_bare.filesystems: - if isinstance(fs, NtfsFilesystem): - usnjrnl_records += len(list(fs.ntfs.usnjrnl.records())) - assert usnjrnl_records == 2 - assert len(target_bare.filesystems) == 4 - assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() - assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" - - @pytest.mark.parametrize( "sub_dir", ["mft", "ntfs", "ntfs_vss", "lazy_ntfs", "auto"], @@ -107,37 +73,13 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> @pytest.mark.parametrize( "paths", [ - (["uploads/file/etc", "uploads/file/var"]), - (["uploads/auto/etc", "uploads/auto/var"]), - (["uploads/file/etc", "uploads/file/var", "uploads/file/opt"]), - (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/opt"]), - (["uploads/file/Library", "uploads/file/Applications"]), - (["uploads/auto/Library", "uploads/auto/Applications"]), - ], -) -def test_unix(paths: list[str], target_bare: Target, tmp_path: Path) -> None: - root = tmp_path - mkdirs(root, paths) - - (root / "uploads.json").write_bytes(b"{}") - - assert VelociraptorLoader.detect(root) is True - - loader = VelociraptorLoader(root) - loader.map(target_bare) - - assert len(target_bare.filesystems) == 1 - - -@pytest.mark.parametrize( - "paths", - [ - (["uploads/file/etc", "uploads/file/var"]), - (["uploads/auto/etc", "uploads/auto/var"]), - (["uploads/file/etc", "uploads/file/var", "uploads/file/opt"]), - (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/opt"]), - (["uploads/file/Library", "uploads/file/Applications"]), - (["uploads/auto/Library", "uploads/auto/Applications"]), + (["uploads/file/etc", "uploads/file/var", "uploads/file/%2ETEST"]), + (["uploads/file/etc", "uploads/file/var", "uploads/file/%2ETEST"]), + (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/%2ETEST"]), + (["uploads/file/etc", "uploads/file/var", "uploads/file/opt", "uploads/file/%2ETEST"]), + (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/opt", "uploads/auto/%2ETEST"]), + (["uploads/file/Library", "uploads/file/Applications", "uploads/file/%2ETEST"]), + (["uploads/auto/Library", "uploads/auto/Applications", "uploads/auto/%2ETEST"]), ], ) def test_unix_zip(paths: list[str], target_bare: Target, tmp_path: Path) -> None: @@ -153,5 +95,7 @@ def test_unix_zip(paths: list[str], target_bare: Target, tmp_path: Path) -> None loader = VelociraptorLoader(zip_path) loader.map(target_bare) + target_bare.apply() assert len(target_bare.filesystems) == 1 + assert target_bare.fs.path("/.TEST").exists() From d69abb0d47006ebf917e04d980614a81bb6c144d Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Sun, 5 May 2024 23:31:25 +0200 Subject: [PATCH 03/26] Adjust comment --- dissect/target/loaders/velociraptor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 41d82ad13..17f9a6bf1 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -77,8 +77,8 @@ class VelociraptorLoader(DirLoader): Generic.Collectors.File (Windows) and Windows.KapeFiles.Targets (Windows) uses the accessors mft, ntfs, lazy_ntfs, ntfs_vss and auto. The loader supports a collection where multiple accessors were used. - The filesystem entries collected by Velociraptor are URL-encoded, which are decoded by the ZIP loader in order to - prevent errors when executing plugins. + The names of the filesystem entries collected by Velociraptor are URL-encoded, which are decoded by the ZIP loader + in order to prevent errors when executing plugins. References: - https://www.rapid7.com/products/velociraptor/ From 934023c41cdb7c01bfe763fd1f409d7841a06cf1 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 8 May 2024 14:25:21 +0200 Subject: [PATCH 04/26] Revert the loading of a directory as target --- dissect/target/loaders/velociraptor.py | 17 +++++--- tests/loaders/test_velociraptor.py | 59 ++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 17f9a6bf1..b5a7d1aea 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -77,9 +77,6 @@ class VelociraptorLoader(DirLoader): Generic.Collectors.File (Windows) and Windows.KapeFiles.Targets (Windows) uses the accessors mft, ntfs, lazy_ntfs, ntfs_vss and auto. The loader supports a collection where multiple accessors were used. - The names of the filesystem entries collected by Velociraptor are URL-encoded, which are decoded by the ZIP loader - in order to prevent errors when executing plugins. - References: - https://www.rapid7.com/products/velociraptor/ - https://docs.velociraptor.app/ @@ -97,6 +94,8 @@ def __init__(self, path: Path, **kwargs): f"Velociraptor target {path!r} is compressed, which will slightly affect performance. " "Consider uncompressing the archive and passing the uncompressed folder to Dissect." ) + else: + self.root = path @staticmethod def detect(path: Path) -> bool: @@ -111,15 +110,19 @@ def detect(path: Path) -> bool: if path.suffix == ".zip": # novermin path = zipfile.Path(path) - if path.joinpath(FILESYSTEMS_ROOT).exists() and path.joinpath("uploads.json").exists(): - _, dirs = find_fs_directories(path) - return bool(dirs) + if path.joinpath(FILESYSTEMS_ROOT).exists() and path.joinpath("uploads.json").exists(): + _, dirs = find_fs_directories(path) + return bool(dirs) return False def map(self, target: Target) -> None: os_type, dirs = find_fs_directories(self.root) if os_type == OperatingSystem.WINDOWS: - map_dirs(target, dirs, os_type, decode_name=True) + if self.path.suffix == ".zip": + map_dirs(target, dirs, os_type, decode_name=True) + else: + # Velociraptor doesn't have the correct filenames for the paths "$J" and "$Secure:$SDS" + map_dirs(target, dirs, os_type, usnjrnl_path="$Extend/$UsnJrnl%3A$J", sds_path="$Secure%3A$SDS") else: map_dirs(target, dirs, os_type, decode_name=True) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 0c8c1d402..3b45d5a8d 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -42,6 +42,40 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: return root +@pytest.mark.parametrize( + "sub_dir, other_dir", + [ + ("mft", "auto"), + ("ntfs", "auto"), + ("ntfs_vss", "auto"), + ("lazy_ntfs", "auto"), + ("auto", "ntfs"), + ], +) +def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_path: Path) -> None: + root = create_root(sub_dir, tmp_path) + root.joinpath(f"uploads/{other_dir}/C%3A").mkdir(parents=True, exist_ok=True) + root.joinpath(f"uploads/{other_dir}/C%3A/other.txt").write_text("my first file") + + assert VelociraptorLoader.detect(root) is True + + loader = VelociraptorLoader(root) + loader.map(target_bare) + target_bare.apply() + + assert "sysvol" in target_bare.fs.mounts + assert "c:" in target_bare.fs.mounts + + usnjrnl_records = 0 + for fs in target_bare.filesystems: + if isinstance(fs, NtfsFilesystem): + usnjrnl_records += len(list(fs.ntfs.usnjrnl.records())) + assert usnjrnl_records == 2 + assert len(target_bare.filesystems) == 4 + assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() + assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" + + @pytest.mark.parametrize( "sub_dir", ["mft", "ntfs", "ntfs_vss", "lazy_ntfs", "auto"], @@ -70,6 +104,31 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() +@pytest.mark.parametrize( + "paths", + [ + (["uploads/file/etc", "uploads/file/var"]), + (["uploads/auto/etc", "uploads/auto/var"]), + (["uploads/file/etc", "uploads/file/var", "uploads/file/opt"]), + (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/opt"]), + (["uploads/file/Library", "uploads/file/Applications"]), + (["uploads/auto/Library", "uploads/auto/Applications"]), + ], +) +def test_unix(paths: list[str], target_bare: Target, tmp_path: Path) -> None: + root = tmp_path + mkdirs(root, paths) + + (root / "uploads.json").write_bytes(b"{}") + + assert VelociraptorLoader.detect(root) is True + + loader = VelociraptorLoader(root) + loader.map(target_bare) + + assert len(target_bare.filesystems) == 1 + + @pytest.mark.parametrize( "paths", [ From 432fc72143144eebf945c79051b8dd74c7d9783e Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 6 Aug 2024 15:45:19 +0200 Subject: [PATCH 05/26] Revert change Will be fixed by https://github.com/fox-it/dissect.target/pull/793 --- dissect/target/filesystems/zip.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dissect/target/filesystems/zip.py b/dissect/target/filesystems/zip.py index b18fc3ca1..109957386 100644 --- a/dissect/target/filesystems/zip.py +++ b/dissect/target/filesystems/zip.py @@ -63,10 +63,7 @@ def __init__( if decode_name: rel_name = unquote(rel_name) - try: - self._fs.map_file_entry(rel_name, file_entry) - except Exception: - log.warning("Failed to map %s into VFS", file_entry) + self._fs.map_file_entry(rel_name, file_entry) @staticmethod def _detect(fh: BinaryIO) -> bool: From 25201a816ed23c803d40a201f12c3dfa1e0709e9 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 6 Aug 2024 16:09:41 +0200 Subject: [PATCH 06/26] Separate mapping of filesystem entries (ZIP) --- dissect/target/filesystems/zip.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/dissect/target/filesystems/zip.py b/dissect/target/filesystems/zip.py index 109957386..8c4d916f0 100644 --- a/dissect/target/filesystems/zip.py +++ b/dissect/target/filesystems/zip.py @@ -48,6 +48,15 @@ def __init__( self._fs = VirtualFilesystem(alt_separator=self.alt_separator, case_sensitive=self.case_sensitive) + self.map_members(decode_name) + + @staticmethod + def _detect(fh: BinaryIO) -> bool: + """Detect a zip file on a given file-like object.""" + return zipfile.is_zipfile(fh) + + def map_members(self, decode_name: bool) -> None: + """Map members of the zip file into the VFS.""" for member in self.zip.infolist(): mname = member.filename.strip("/") if not mname.startswith(self.base) or mname == ".": @@ -65,10 +74,6 @@ def __init__( self._fs.map_file_entry(rel_name, file_entry) - @staticmethod - def _detect(fh: BinaryIO) -> bool: - """Detect a zip file on a given file-like object.""" - return zipfile.is_zipfile(fh) def get(self, path: str, relentry: FilesystemEntry = None) -> FilesystemEntry: """Returns a ZipFilesystemEntry object corresponding to the given path.""" From 57a038a89e591320791b0438f5e457900204aefe Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 14 Aug 2024 13:36:15 +0200 Subject: [PATCH 07/26] Fix merge and change name of argument --- dissect/target/filesystems/zip.py | 15 +++++---------- dissect/target/loaders/dir.py | 15 ++++++++++++--- dissect/target/loaders/velociraptor.py | 17 +++++++++-------- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/dissect/target/filesystems/zip.py b/dissect/target/filesystems/zip.py index cfe7c0702..fa62c572b 100644 --- a/dissect/target/filesystems/zip.py +++ b/dissect/target/filesystems/zip.py @@ -4,12 +4,8 @@ import stat import zipfile from datetime import datetime, timezone -<<<<<<< HEAD -from typing import BinaryIO, Optional -from urllib.parse import unquote -======= from typing import BinaryIO, Iterator ->>>>>>> upstream/main +from urllib.parse import unquote from dissect.util.stream import BufferedStream @@ -44,7 +40,7 @@ def __init__( self, fh: BinaryIO, base: str | None = None, - decode_name: bool = False, + unquote_path: bool = False, *args, **kwargs, ): @@ -57,14 +53,14 @@ def __init__( self._fs = VirtualFilesystem(alt_separator=self.alt_separator, case_sensitive=self.case_sensitive) - self.map_members(decode_name) + self.map_members(unquote_path) @staticmethod def _detect(fh: BinaryIO) -> bool: """Detect a zip file on a given file-like object.""" return zipfile.is_zipfile(fh) - def map_members(self, decode_name: bool) -> None: + def map_members(self, unquote_path: bool) -> None: """Map members of the zip file into the VFS.""" for member in self.zip.infolist(): mname = member.filename.strip("/") @@ -73,12 +69,11 @@ def map_members(self, decode_name: bool) -> None: rel_name = fsutil.normpath(mname[len(self.base) :], alt_separator=self.alt_separator) - if decode_name: + if unquote_path: rel_name = unquote(rel_name) self._fs.map_file_entry(rel_name, ZipFilesystemEntry(self, rel_name, member)) - def get(self, path: str, relentry: FilesystemEntry = None) -> FilesystemEntry: """Returns a ZipFilesystemEntry object corresponding to the given path.""" return self._fs.get(path, relentry=relentry) diff --git a/dissect/target/loaders/dir.py b/dissect/target/loaders/dir.py index 497875235..7321f076b 100644 --- a/dissect/target/loaders/dir.py +++ b/dissect/target/loaders/dir.py @@ -37,7 +37,11 @@ def find_entry_path(path: Path) -> str | None: def map_dirs( - target: Target, dirs: list[Path | tuple[str, Path]], os_type: str, decode_name: bool = False, **kwargs + target: Target, + dirs: list[Path | tuple[str, Path]], + os_type: str, + unquote_path: bool = False, + **kwargs, ) -> None: """Map directories as filesystems into the given target. @@ -66,10 +70,15 @@ def map_dirs( path.at, alt_separator=alt_separator, case_sensitive=case_sensitive, - decode_name=decode_name, + unquote_path=unquote_path, ) else: - dfs = DirectoryFilesystem(path, alt_separator=alt_separator, case_sensitive=case_sensitive) + dfs = DirectoryFilesystem( + path, + alt_separator=alt_separator, + case_sensitive=case_sensitive, + unquote_path=unquote_path, + ) drive_letter_map[drive_letter].append(dfs) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index b5a7d1aea..56b365e24 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -118,11 +118,12 @@ def detect(path: Path) -> bool: def map(self, target: Target) -> None: os_type, dirs = find_fs_directories(self.root) - if os_type == OperatingSystem.WINDOWS: - if self.path.suffix == ".zip": - map_dirs(target, dirs, os_type, decode_name=True) - else: - # Velociraptor doesn't have the correct filenames for the paths "$J" and "$Secure:$SDS" - map_dirs(target, dirs, os_type, usnjrnl_path="$Extend/$UsnJrnl%3A$J", sds_path="$Secure%3A$SDS") - else: - map_dirs(target, dirs, os_type, decode_name=True) + + # Velociraptor URL encodes paths before storing these in a zip file, this leads plugins not being able to find + # these paths. To prevent this issue, the path names are URL decoded before mapping into the VFS. + map_dirs( + target, + dirs, + os_type, + unquote_path=True, + ) From 78eb3f6fc4d2888643369fa4d197423a90521102 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 14 Aug 2024 14:31:51 +0200 Subject: [PATCH 08/26] Add tests --- tests/loaders/test_velociraptor.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 3b45d5a8d..9d9d2513f 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -107,12 +107,12 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> @pytest.mark.parametrize( "paths", [ - (["uploads/file/etc", "uploads/file/var"]), - (["uploads/auto/etc", "uploads/auto/var"]), - (["uploads/file/etc", "uploads/file/var", "uploads/file/opt"]), - (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/opt"]), - (["uploads/file/Library", "uploads/file/Applications"]), - (["uploads/auto/Library", "uploads/auto/Applications"]), + (["uploads/file/etc", "uploads/file/var", "uploads/file/%2ETEST"]), + (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/%2ETEST"]), + (["uploads/file/etc", "uploads/file/var", "uploads/file/opt", "uploads/file/%2ETEST"]), + (["uploads/auto/etc", "uploads/auto/var", "uploads/auto/opt", "uploads/auto/%2ETEST"]), + (["uploads/file/Library", "uploads/file/Applications", "uploads/file/%2ETEST"]), + (["uploads/auto/Library", "uploads/auto/Applications", "uploads/auto/%2ETEST"]), ], ) def test_unix(paths: list[str], target_bare: Target, tmp_path: Path) -> None: @@ -125,8 +125,10 @@ def test_unix(paths: list[str], target_bare: Target, tmp_path: Path) -> None: loader = VelociraptorLoader(root) loader.map(target_bare) + target_bare.apply() assert len(target_bare.filesystems) == 1 + assert target_bare.fs.path("/.TEST").exists() @pytest.mark.parametrize( From 31fc6d4eb80d0804b14d212c5bda54d10f83d165 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 14 Aug 2024 15:51:10 +0200 Subject: [PATCH 09/26] Use VFS for the `DirectoryFilesystem` --- dissect/target/filesystems/dir.py | 57 +++++++++++++++++------------- tests/loaders/test_velociraptor.py | 3 ++ 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/dissect/target/filesystems/dir.py b/dissect/target/filesystems/dir.py index 998bbe87a..c91633331 100644 --- a/dissect/target/filesystems/dir.py +++ b/dissect/target/filesystems/dir.py @@ -1,25 +1,34 @@ import os from pathlib import Path from typing import BinaryIO, Iterator +from urllib.parse import unquote from dissect.target.exceptions import ( - FileNotFoundError, FilesystemError, IsADirectoryError, NotADirectoryError, NotASymlinkError, ) -from dissect.target.filesystem import Filesystem, FilesystemEntry +from dissect.target.filesystem import ( + Filesystem, + FilesystemEntry, + VirtualDirectory, + VirtualFilesystem, +) from dissect.target.helpers import fsutil class DirectoryFilesystem(Filesystem): __type__ = "dir" - def __init__(self, path: Path, *args, **kwargs): + def __init__(self, path: Path, unquote_path: bool = False, *args, **kwargs): super().__init__(None, *args, **kwargs) self.base_path = path + self._fs = VirtualFilesystem(alt_separator=self.alt_separator, case_sensitive=self.case_sensitive) + + self.map_members(unquote_path) + def __repr__(self) -> str: return f"<{self.__class__.__name__} {self.base_path}>" @@ -27,37 +36,37 @@ def __repr__(self) -> str: def _detect(fh: BinaryIO) -> bool: raise TypeError("Detect is not allowed on DirectoryFilesystem class") - def get(self, path: str) -> FilesystemEntry: - path = path.strip("/") - - if not path: - return DirectoryFilesystemEntry(self, "/", self.base_path) + def map_members(self, unquote_path: bool) -> None: + """Map members of a path into the VFS.""" - if not self.case_sensitive: - searchpath = self.base_path + for path in self.base_path.rglob("*"): + pname = str(path) - for p in path.split("/"): - match = [d for d in searchpath.iterdir() if d.name.lower() == p.lower()] + if not pname.startswith(str(self.base_path)) or pname == ".": + continue - if not match or len(match) > 1: - raise FileNotFoundError(path) + rel_name = fsutil.normpath(pname[len(str(self.base_path)) :], alt_separator=self.alt_separator) - searchpath = match[0] + if unquote_path: + rel_name = unquote(rel_name) - entry = searchpath - else: - entry = self.base_path.joinpath(path.strip("/")) + self._fs.map_file_entry(rel_name, DirectoryFilesystemEntry(self, rel_name, path)) - try: - entry.lstat() - return DirectoryFilesystemEntry(self, path, entry) - except Exception: - raise FileNotFoundError(path) + def get(self, path: str, relentry: FilesystemEntry = None) -> FilesystemEntry: + """Returns a FilesystemEntry object corresponding to the given path.""" + return self._fs.get(path, relentry=relentry) -class DirectoryFilesystemEntry(FilesystemEntry): +# Note: We subclass from VirtualDirectory because VirtualFilesystem is currently only compatible with VirtualDirectory +# Subclass from VirtualDirectory so we get that compatibility for free, and override the rest to do our own thing +class DirectoryFilesystemEntry(VirtualDirectory): + fs: DirectoryFilesystem entry: Path + def __init__(self, fs: DirectoryFilesystem, path: str, entry: Path): + super().__init__(fs, path) + self.entry = entry + def get(self, path: str) -> FilesystemEntry: path = fsutil.join(self.path, path, alt_separator=self.fs.alt_separator) return self.fs.get(path) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 9d9d2513f..709deff11 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -17,6 +17,7 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/", f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/$Extend", f"uploads/{sub_dir}/%5C%5C%3F%5CGLOBALROOT%5CDevice%5CHarddiskVolumeShadowCopy1/windows/system32", + f"uploads/{sub_dir}/%5C%5C.%5CC%3A/%2ETEST", ] root = tmp_path mkdirs(root, paths) @@ -74,6 +75,7 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat assert len(target_bare.filesystems) == 4 assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" + assert target_bare.fs.path("sysvol/.TEST").exists() @pytest.mark.parametrize( @@ -102,6 +104,7 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> assert usnjrnl_records == 2 assert len(target_bare.filesystems) == 4 assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() + assert target_bare.fs.path("sysvol/.TEST").exists() @pytest.mark.parametrize( From 5528a1d50137db3cc746785173f697c0938b3134 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 21 Aug 2024 12:34:13 +0200 Subject: [PATCH 10/26] Revert changes --- dissect/target/filesystems/dir.py | 57 +++++++++++++------------------ 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/dissect/target/filesystems/dir.py b/dissect/target/filesystems/dir.py index c91633331..998bbe87a 100644 --- a/dissect/target/filesystems/dir.py +++ b/dissect/target/filesystems/dir.py @@ -1,34 +1,25 @@ import os from pathlib import Path from typing import BinaryIO, Iterator -from urllib.parse import unquote from dissect.target.exceptions import ( + FileNotFoundError, FilesystemError, IsADirectoryError, NotADirectoryError, NotASymlinkError, ) -from dissect.target.filesystem import ( - Filesystem, - FilesystemEntry, - VirtualDirectory, - VirtualFilesystem, -) +from dissect.target.filesystem import Filesystem, FilesystemEntry from dissect.target.helpers import fsutil class DirectoryFilesystem(Filesystem): __type__ = "dir" - def __init__(self, path: Path, unquote_path: bool = False, *args, **kwargs): + def __init__(self, path: Path, *args, **kwargs): super().__init__(None, *args, **kwargs) self.base_path = path - self._fs = VirtualFilesystem(alt_separator=self.alt_separator, case_sensitive=self.case_sensitive) - - self.map_members(unquote_path) - def __repr__(self) -> str: return f"<{self.__class__.__name__} {self.base_path}>" @@ -36,37 +27,37 @@ def __repr__(self) -> str: def _detect(fh: BinaryIO) -> bool: raise TypeError("Detect is not allowed on DirectoryFilesystem class") - def map_members(self, unquote_path: bool) -> None: - """Map members of a path into the VFS.""" + def get(self, path: str) -> FilesystemEntry: + path = path.strip("/") + + if not path: + return DirectoryFilesystemEntry(self, "/", self.base_path) - for path in self.base_path.rglob("*"): - pname = str(path) + if not self.case_sensitive: + searchpath = self.base_path - if not pname.startswith(str(self.base_path)) or pname == ".": - continue + for p in path.split("/"): + match = [d for d in searchpath.iterdir() if d.name.lower() == p.lower()] - rel_name = fsutil.normpath(pname[len(str(self.base_path)) :], alt_separator=self.alt_separator) + if not match or len(match) > 1: + raise FileNotFoundError(path) - if unquote_path: - rel_name = unquote(rel_name) + searchpath = match[0] - self._fs.map_file_entry(rel_name, DirectoryFilesystemEntry(self, rel_name, path)) + entry = searchpath + else: + entry = self.base_path.joinpath(path.strip("/")) - def get(self, path: str, relentry: FilesystemEntry = None) -> FilesystemEntry: - """Returns a FilesystemEntry object corresponding to the given path.""" - return self._fs.get(path, relentry=relentry) + try: + entry.lstat() + return DirectoryFilesystemEntry(self, path, entry) + except Exception: + raise FileNotFoundError(path) -# Note: We subclass from VirtualDirectory because VirtualFilesystem is currently only compatible with VirtualDirectory -# Subclass from VirtualDirectory so we get that compatibility for free, and override the rest to do our own thing -class DirectoryFilesystemEntry(VirtualDirectory): - fs: DirectoryFilesystem +class DirectoryFilesystemEntry(FilesystemEntry): entry: Path - def __init__(self, fs: DirectoryFilesystem, path: str, entry: Path): - super().__init__(fs, path) - self.entry = entry - def get(self, path: str) -> FilesystemEntry: path = fsutil.join(self.path, path, alt_separator=self.fs.alt_separator) return self.fs.get(path) From 26197a94866e54377adbd5f6a949bbe5f043193b Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Mon, 26 Aug 2024 18:16:55 +0200 Subject: [PATCH 11/26] Don't reopen the archive for the compression type --- dissect/target/loaders/velociraptor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index bc8477980..aef9ee04e 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -88,7 +88,7 @@ def __init__(self, path: Path, **kwargs): if path.suffix == ".zip": self.root = zipfile.Path(path.open("rb")) - compression_type = zipfile.ZipFile(str(path)).getinfo("uploads.json").compress_type + compression_type = self.root.root.getinfo("uploads.json").compress_type if compression_type > 0: log.warning( f"Velociraptor target {path!r} is compressed, which will slightly affect performance. " From 84a9cc0b6c2c179174cd4aedaf063cf13a362d5e Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 28 Aug 2024 11:33:06 +0200 Subject: [PATCH 12/26] Add VelociraptorDirectoryFilesystem --- dissect/target/filesystems/dir.py | 17 +++++++++++------ dissect/target/filesystems/velociraptor.py | 14 ++++++++++++++ dissect/target/loaders/dir.py | 16 +++++++--------- dissect/target/loaders/velociraptor.py | 10 ++++++++-- tests/loaders/test_velociraptor.py | 3 +-- 5 files changed, 41 insertions(+), 19 deletions(-) create mode 100644 dissect/target/filesystems/velociraptor.py diff --git a/dissect/target/filesystems/dir.py b/dissect/target/filesystems/dir.py index 998bbe87a..811149256 100644 --- a/dissect/target/filesystems/dir.py +++ b/dissect/target/filesystems/dir.py @@ -27,12 +27,7 @@ def __repr__(self) -> str: def _detect(fh: BinaryIO) -> bool: raise TypeError("Detect is not allowed on DirectoryFilesystem class") - def get(self, path: str) -> FilesystemEntry: - path = path.strip("/") - - if not path: - return DirectoryFilesystemEntry(self, "/", self.base_path) - + def _resolve_path(self, path: str) -> Path: if not self.case_sensitive: searchpath = self.base_path @@ -48,6 +43,16 @@ def get(self, path: str) -> FilesystemEntry: else: entry = self.base_path.joinpath(path.strip("/")) + return entry + + def get(self, path: str) -> FilesystemEntry: + path = path.strip("/") + + if not path: + return DirectoryFilesystemEntry(self, "/", self.base_path) + + entry = self._resolve_path(path) + try: entry.lstat() return DirectoryFilesystemEntry(self, path, entry) diff --git a/dissect/target/filesystems/velociraptor.py b/dissect/target/filesystems/velociraptor.py new file mode 100644 index 000000000..a504e7f7b --- /dev/null +++ b/dissect/target/filesystems/velociraptor.py @@ -0,0 +1,14 @@ +from pathlib import Path +from urllib.parse import unquote + +from dissect.target.filesystems.dir import DirectoryFilesystem + + +class VelociraptorDirectoryFilesystem(DirectoryFilesystem): + __type__ = "velociraptor_dir" + + def __init__(self, path: Path, *args, **kwargs): + super().__init__(path, *args, **kwargs) + + def _resolve_path(self, path) -> Path: + return super()._resolve_path(unquote(path).replace(".", "%2E")) diff --git a/dissect/target/loaders/dir.py b/dissect/target/loaders/dir.py index 7321f076b..1516ed0ec 100644 --- a/dissect/target/loaders/dir.py +++ b/dissect/target/loaders/dir.py @@ -3,11 +3,10 @@ import zipfile from collections import defaultdict from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Callable from dissect.target.filesystem import LayerFilesystem from dissect.target.filesystems.dir import DirectoryFilesystem -from dissect.target.filesystems.zip import ZipFilesystem from dissect.target.helpers import loaderutil from dissect.target.loader import Loader from dissect.target.plugin import OperatingSystem @@ -40,6 +39,7 @@ def map_dirs( target: Target, dirs: list[Path | tuple[str, Path]], os_type: str, + fs_type: Callable = DirectoryFilesystem, unquote_path: bool = False, **kwargs, ) -> None: @@ -49,6 +49,9 @@ def map_dirs( target: The target to map into. dirs: The directories to map as filesystems. If a list member is a tuple, the first element is the drive letter. os_type: The operating system type, used to determine how the filesystem should be mounted. + fs_type: The filesystem type: + unquote_path: ``True`` URL decodes the ``path``, ``False`` it does not, + if the ``ZipFilesystem`` is specified as ``fs_type``. """ alt_separator = "" case_sensitive = True @@ -65,7 +68,7 @@ def map_dirs( drive_letter = path.name[0] if isinstance(path, zipfile.Path): - dfs = ZipFilesystem( + dfs = fs_type( path.root.fp, path.at, alt_separator=alt_separator, @@ -73,12 +76,7 @@ def map_dirs( unquote_path=unquote_path, ) else: - dfs = DirectoryFilesystem( - path, - alt_separator=alt_separator, - case_sensitive=case_sensitive, - unquote_path=unquote_path, - ) + dfs = fs_type(path, alt_separator=alt_separator, case_sensitive=case_sensitive) drive_letter_map[drive_letter].append(dfs) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index aef9ee04e..fe1256df3 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -5,6 +5,8 @@ from pathlib import Path from typing import TYPE_CHECKING +from dissect.target.filesystems.velociraptor import VelociraptorDirectoryFilesystem +from dissect.target.filesystems.zip import ZipFilesystem from dissect.target.loaders.dir import DirLoader, find_dirs, map_dirs from dissect.target.plugin import OperatingSystem @@ -88,6 +90,7 @@ def __init__(self, path: Path, **kwargs): if path.suffix == ".zip": self.root = zipfile.Path(path.open("rb")) + self.fs_type = ZipFilesystem compression_type = self.root.root.getinfo("uploads.json").compress_type if compression_type > 0: log.warning( @@ -96,6 +99,7 @@ def __init__(self, path: Path, **kwargs): ) else: self.root = path + self.fs_type = VelociraptorDirectoryFilesystem @staticmethod def detect(path: Path) -> bool: @@ -119,11 +123,13 @@ def detect(path: Path) -> bool: def map(self, target: Target) -> None: os_type, dirs = find_fs_directories(self.root) - # Velociraptor URL encodes paths before storing these in a zip file, this leads plugins not being able to find - # these paths. To prevent this issue, the path names are URL decoded before mapping into the VFS. + # Velociraptor URL encodes paths before storing these in a collection, this leads plugins not being able to find + # these paths. To circumvent this issue, for a zip file the path names are URL decoded before mapping into the + # VFS and for a directory the paths are URL encoded at lookup time. map_dirs( target, dirs, os_type, + self.fs_type, unquote_path=True, ) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 709deff11..db847501c 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -24,6 +24,7 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: (root / "uploads.json").write_bytes(b"{}") (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/C-DRIVE.txt").write_bytes(b"{}") + (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/other.txt").write_text("my first file") with open(absolute_path("_data/plugins/filesystem/ntfs/mft/mft.raw"), "rb") as fh: mft = fh.read(10 * 1025) @@ -55,8 +56,6 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: ) def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_path: Path) -> None: root = create_root(sub_dir, tmp_path) - root.joinpath(f"uploads/{other_dir}/C%3A").mkdir(parents=True, exist_ok=True) - root.joinpath(f"uploads/{other_dir}/C%3A/other.txt").write_text("my first file") assert VelociraptorLoader.detect(root) is True From fdaee1f471a58ffb2a5761215ed2e0404947f76e Mon Sep 17 00:00:00 2001 From: Schamper <1254028+Schamper@users.noreply.github.com> Date: Wed, 28 Aug 2024 12:39:00 +0200 Subject: [PATCH 13/26] Suggestions --- dissect/target/filesystems/velociraptor.py | 14 ----------- dissect/target/filesystems/zip.py | 23 +++++++----------- dissect/target/loaders/dir.py | 23 ++++++++---------- dissect/target/loaders/velociraptor.py | 27 ++++++++++++++-------- 4 files changed, 35 insertions(+), 52 deletions(-) delete mode 100644 dissect/target/filesystems/velociraptor.py diff --git a/dissect/target/filesystems/velociraptor.py b/dissect/target/filesystems/velociraptor.py deleted file mode 100644 index a504e7f7b..000000000 --- a/dissect/target/filesystems/velociraptor.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path -from urllib.parse import unquote - -from dissect.target.filesystems.dir import DirectoryFilesystem - - -class VelociraptorDirectoryFilesystem(DirectoryFilesystem): - __type__ = "velociraptor_dir" - - def __init__(self, path: Path, *args, **kwargs): - super().__init__(path, *args, **kwargs) - - def _resolve_path(self, path) -> Path: - return super()._resolve_path(unquote(path).replace(".", "%2E")) diff --git a/dissect/target/filesystems/zip.py b/dissect/target/filesystems/zip.py index fa62c572b..f525369e1 100644 --- a/dissect/target/filesystems/zip.py +++ b/dissect/target/filesystems/zip.py @@ -5,7 +5,6 @@ import zipfile from datetime import datetime, timezone from typing import BinaryIO, Iterator -from urllib.parse import unquote from dissect.util.stream import BufferedStream @@ -40,7 +39,6 @@ def __init__( self, fh: BinaryIO, base: str | None = None, - unquote_path: bool = False, *args, **kwargs, ): @@ -53,26 +51,21 @@ def __init__( self._fs = VirtualFilesystem(alt_separator=self.alt_separator, case_sensitive=self.case_sensitive) - self.map_members(unquote_path) - - @staticmethod - def _detect(fh: BinaryIO) -> bool: - """Detect a zip file on a given file-like object.""" - return zipfile.is_zipfile(fh) - - def map_members(self, unquote_path: bool) -> None: - """Map members of the zip file into the VFS.""" for member in self.zip.infolist(): mname = member.filename.strip("/") if not mname.startswith(self.base) or mname == ".": continue - rel_name = fsutil.normpath(mname[len(self.base) :], alt_separator=self.alt_separator) + rel_name = self._resolve_path(mname) + self._fs.map_file_entry(rel_name, ZipFilesystemEntry(self, rel_name, member)) - if unquote_path: - rel_name = unquote(rel_name) + @staticmethod + def _detect(fh: BinaryIO) -> bool: + """Detect a zip file on a given file-like object.""" + return zipfile.is_zipfile(fh) - self._fs.map_file_entry(rel_name, ZipFilesystemEntry(self, rel_name, member)) + def _resolve_path(self, path: str) -> str: + return fsutil.normpath(path[len(self.base) :], alt_separator=self.alt_separator) def get(self, path: str, relentry: FilesystemEntry = None) -> FilesystemEntry: """Returns a ZipFilesystemEntry object corresponding to the given path.""" diff --git a/dissect/target/loaders/dir.py b/dissect/target/loaders/dir.py index 1516ed0ec..ed279f658 100644 --- a/dissect/target/loaders/dir.py +++ b/dissect/target/loaders/dir.py @@ -3,10 +3,11 @@ import zipfile from collections import defaultdict from pathlib import Path -from typing import TYPE_CHECKING, Callable +from typing import TYPE_CHECKING from dissect.target.filesystem import LayerFilesystem from dissect.target.filesystems.dir import DirectoryFilesystem +from dissect.target.filesystems.zip import ZipFilesystem from dissect.target.helpers import loaderutil from dissect.target.loader import Loader from dissect.target.plugin import OperatingSystem @@ -39,8 +40,9 @@ def map_dirs( target: Target, dirs: list[Path | tuple[str, Path]], os_type: str, - fs_type: Callable = DirectoryFilesystem, - unquote_path: bool = False, + *, + dirfs: type[DirectoryFilesystem] = DirectoryFilesystem, + zipfs: type[ZipFilesystem] = ZipFilesystem, **kwargs, ) -> None: """Map directories as filesystems into the given target. @@ -49,9 +51,8 @@ def map_dirs( target: The target to map into. dirs: The directories to map as filesystems. If a list member is a tuple, the first element is the drive letter. os_type: The operating system type, used to determine how the filesystem should be mounted. - fs_type: The filesystem type: - unquote_path: ``True`` URL decodes the ``path``, ``False`` it does not, - if the ``ZipFilesystem`` is specified as ``fs_type``. + dirfs: The filesystem class to use for directory filesystems. + zipfs: The filesystem class to use for ZIP filesystems. """ alt_separator = "" case_sensitive = True @@ -68,15 +69,9 @@ def map_dirs( drive_letter = path.name[0] if isinstance(path, zipfile.Path): - dfs = fs_type( - path.root.fp, - path.at, - alt_separator=alt_separator, - case_sensitive=case_sensitive, - unquote_path=unquote_path, - ) + dfs = zipfs(path.root.fp, path.at, alt_separator=alt_separator, case_sensitive=case_sensitive) else: - dfs = fs_type(path, alt_separator=alt_separator, case_sensitive=case_sensitive) + dfs = dirfs(path, alt_separator=alt_separator, case_sensitive=case_sensitive) drive_letter_map[drive_letter].append(dfs) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index fe1256df3..20a5d609e 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -4,8 +4,9 @@ import zipfile from pathlib import Path from typing import TYPE_CHECKING +from urllib.parse import unquote -from dissect.target.filesystems.velociraptor import VelociraptorDirectoryFilesystem +from dissect.target.filesystems.dir import DirectoryFilesystem from dissect.target.filesystems.zip import ZipFilesystem from dissect.target.loaders.dir import DirLoader, find_dirs, map_dirs from dissect.target.plugin import OperatingSystem @@ -90,16 +91,14 @@ def __init__(self, path: Path, **kwargs): if path.suffix == ".zip": self.root = zipfile.Path(path.open("rb")) - self.fs_type = ZipFilesystem - compression_type = self.root.root.getinfo("uploads.json").compress_type - if compression_type > 0: + if self.root.root.getinfo("uploads.json").compress_type > 0: log.warning( - f"Velociraptor target {path!r} is compressed, which will slightly affect performance. " - "Consider uncompressing the archive and passing the uncompressed folder to Dissect." + "Velociraptor target '%s' is compressed, which will slightly affect performance. " + "Consider uncompressing the archive and passing the uncompressed folder to Dissect.", + path, ) else: self.root = path - self.fs_type = VelociraptorDirectoryFilesystem @staticmethod def detect(path: Path) -> bool: @@ -130,6 +129,16 @@ def map(self, target: Target) -> None: target, dirs, os_type, - self.fs_type, - unquote_path=True, + dirfs=VelociraptorDirectoryFilesystem, + zipfs=VelociraptorZipFilesystem, ) + + +class VelociraptorDirectoryFilesystem(DirectoryFilesystem): + def _resolve_path(self, path: str) -> Path: + return super()._resolve_path(unquote(path).replace(".", "%2E")) + + +class VelociraptorZipFilesystem(ZipFilesystem): + def _resolve_path(self, path: str) -> str: + return super()._resolve_path(unquote(path).replace(".", "%2E")) From 3d14d2a21c485f8b5e7498c92fc4b43c61176b3d Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Sat, 28 Sep 2024 23:00:48 +0200 Subject: [PATCH 14/26] Fix loader --- dissect/target/loaders/velociraptor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 20a5d609e..da71585b3 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -4,7 +4,7 @@ import zipfile from pathlib import Path from typing import TYPE_CHECKING -from urllib.parse import unquote +from urllib.parse import quote, unquote from dissect.target.filesystems.dir import DirectoryFilesystem from dissect.target.filesystems.zip import ZipFilesystem @@ -136,9 +136,9 @@ def map(self, target: Target) -> None: class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: - return super()._resolve_path(unquote(path).replace(".", "%2E")) + return super()._resolve_path(quote(path, "$/").replace(".", "%2E")) class VelociraptorZipFilesystem(ZipFilesystem): def _resolve_path(self, path: str) -> str: - return super()._resolve_path(unquote(path).replace(".", "%2E")) + return unquote(super()._resolve_path(path)).replace("%2E", ".") From bc076dabf7bfd7cd3027352a749dc5f23b8ad2bd Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Sun, 29 Sep 2024 15:10:37 +0200 Subject: [PATCH 15/26] Explain bug --- tests/loaders/test_velociraptor.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index db847501c..9ced35291 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -72,8 +72,19 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat usnjrnl_records += len(list(fs.ntfs.usnjrnl.records())) assert usnjrnl_records == 2 assert len(target_bare.filesystems) == 4 - assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() - assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" + + # FIXME: These tests fail, but the test below this works while the objects are the same + # assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() + # assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" + + paths = list(target_bare.fs.path("sysvol").iterdir()) + + assert paths[2].exists() + assert paths[2] == target_bare.fs.path("sysvol/C-DRIVE.txt") + + assert paths[4].exists() + assert paths[4] == target_bare.fs.path("sysvol/other.txt") + assert target_bare.fs.path("sysvol/.TEST").exists() From 2e13ed2c84708c8594dd6a1092674a99f1484c1d Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Mon, 14 Oct 2024 10:39:46 +0200 Subject: [PATCH 16/26] Only replace first occurrence of a string --- dissect/target/loaders/velociraptor.py | 9 +++++++-- tests/loaders/test_velociraptor.py | 5 ++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index da71585b3..769a87524 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -136,9 +136,14 @@ def map(self, target: Target) -> None: class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: - return super()._resolve_path(quote(path, "$/").replace(".", "%2E")) + path = quote(path, "$/") + path = path.replace(".", "%2E", 1) if path.startswith(".") else path + path = super()._resolve_path(path) + return path class VelociraptorZipFilesystem(ZipFilesystem): def _resolve_path(self, path: str) -> str: - return unquote(super()._resolve_path(path)).replace("%2E", ".") + path = unquote(super()._resolve_path(path)) + path = path.replace("%2E", ".", 1) if path.startswith("%2E") else path + return path diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 9ced35291..2def296e8 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -73,9 +73,8 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat assert usnjrnl_records == 2 assert len(target_bare.filesystems) == 4 - # FIXME: These tests fail, but the test below this works while the objects are the same - # assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() - # assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" + assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() + assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" paths = list(target_bare.fs.path("sysvol").iterdir()) From c3b208dee98c423870136af43815a5ce65627638 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Mon, 18 Nov 2024 12:06:04 +0100 Subject: [PATCH 17/26] Apply code review suggestion --- dissect/target/loaders/velociraptor.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 769a87524..91d51fb17 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -136,14 +136,12 @@ def map(self, target: Target) -> None: class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: - path = quote(path, "$/") + path = quote(path, safe="$/") path = path.replace(".", "%2E", 1) if path.startswith(".") else path - path = super()._resolve_path(path) - return path + return super()._resolve_path(path) class VelociraptorZipFilesystem(ZipFilesystem): def _resolve_path(self, path: str) -> str: path = unquote(super()._resolve_path(path)) - path = path.replace("%2E", ".", 1) if path.startswith("%2E") else path - return path + return path.replace("%2E", ".", 1) if path.startswith("%2E") else path From 3a067d0f88e5190b59d79819ee92385ecf332f78 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 19 Nov 2024 10:49:31 +0100 Subject: [PATCH 18/26] Fix ZIP `_resolve_path` `unquote` is enough because it also fixes `root/%2Ebash_history` -> `root/.bash_history`. --- dissect/target/loaders/velociraptor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 91d51fb17..164c76b4e 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -143,5 +143,4 @@ def _resolve_path(self, path: str) -> Path: class VelociraptorZipFilesystem(ZipFilesystem): def _resolve_path(self, path: str) -> str: - path = unquote(super()._resolve_path(path)) - return path.replace("%2E", ".", 1) if path.startswith("%2E") else path + return unquote(super()._resolve_path(path)) From 7922897a9d2513b21626c0f1b16b33658bff90ca Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 19 Nov 2024 10:49:39 +0100 Subject: [PATCH 19/26] Extend tests --- tests/loaders/test_velociraptor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 2def296e8..5d431c45e 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -24,6 +24,7 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: (root / "uploads.json").write_bytes(b"{}") (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/C-DRIVE.txt").write_bytes(b"{}") + (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/Test%254Test.evtx").write_bytes(b"{}") (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/other.txt").write_text("my first file") with open(absolute_path("_data/plugins/filesystem/ntfs/mft/mft.raw"), "rb") as fh: @@ -81,10 +82,12 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat assert paths[2].exists() assert paths[2] == target_bare.fs.path("sysvol/C-DRIVE.txt") - assert paths[4].exists() - assert paths[4] == target_bare.fs.path("sysvol/other.txt") + assert paths[5].exists() + assert paths[5] == target_bare.fs.path("sysvol/other.txt") assert target_bare.fs.path("sysvol/.TEST").exists() + print(paths) + assert target_bare.fs.path("sysvol/Test%4Test.evtx").exists() @pytest.mark.parametrize( @@ -114,6 +117,7 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> assert len(target_bare.filesystems) == 4 assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() assert target_bare.fs.path("sysvol/.TEST").exists() + assert target_bare.fs.path("sysvol/Test%4Test.evtx").exists() @pytest.mark.parametrize( From f73441747a19c581de69a382825adf3759300da8 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 19 Nov 2024 10:50:56 +0100 Subject: [PATCH 20/26] Fix dir `_resolve_path` Only adjust the name of the path --- dissect/target/loaders/velociraptor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 164c76b4e..b054b8b2a 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -136,8 +136,8 @@ def map(self, target: Target) -> None: class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: - path = quote(path, safe="$/") - path = path.replace(".", "%2E", 1) if path.startswith(".") else path + path = Path(quote(path, safe="$/")) + path = str(path.with_name(path.name.replace(".", "%2E", 1))) if path.name.startswith(".") else str(path) return super()._resolve_path(path) From ff14434b0abb4f7790ae7717b18043039b60926c Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 19 Nov 2024 13:24:37 +0100 Subject: [PATCH 21/26] Remove `print` statement --- tests/loaders/test_velociraptor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 5d431c45e..a48f20c51 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -86,7 +86,6 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat assert paths[5] == target_bare.fs.path("sysvol/other.txt") assert target_bare.fs.path("sysvol/.TEST").exists() - print(paths) assert target_bare.fs.path("sysvol/Test%4Test.evtx").exists() From ba1d362cf865a6dc9b96a6551a5135e490c04dd2 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 19 Nov 2024 15:01:16 +0100 Subject: [PATCH 22/26] Apply suggestions code review Skip `% ` due to files that have decoded values by default, for example EVTX files. --- dissect/target/filesystems/dir.py | 2 +- dissect/target/loaders/velociraptor.py | 7 +++++-- tests/loaders/test_velociraptor.py | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dissect/target/filesystems/dir.py b/dissect/target/filesystems/dir.py index 811149256..5ed2aa023 100644 --- a/dissect/target/filesystems/dir.py +++ b/dissect/target/filesystems/dir.py @@ -48,7 +48,7 @@ def _resolve_path(self, path: str) -> Path: def get(self, path: str) -> FilesystemEntry: path = path.strip("/") - if not path: + if not (path := path.strip("/")): return DirectoryFilesystemEntry(self, "/", self.base_path) entry = self._resolve_path(path) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index b054b8b2a..707c9f365 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -8,6 +8,7 @@ from dissect.target.filesystems.dir import DirectoryFilesystem from dissect.target.filesystems.zip import ZipFilesystem +from dissect.target.helpers.fsutil import basename, dirname, join from dissect.target.loaders.dir import DirLoader, find_dirs, map_dirs from dissect.target.plugin import OperatingSystem @@ -136,8 +137,10 @@ def map(self, target: Target) -> None: class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: - path = Path(quote(path, safe="$/")) - path = str(path.with_name(path.name.replace(".", "%2E", 1))) if path.name.startswith(".") else str(path) + path = quote(path, safe="$/% ") + path = ( + join(dirname(path), str(basename(path)).replace(".", "%2E", 1)) if basename(path).startswith(".") else path + ) return super()._resolve_path(path) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index a48f20c51..16d184ddb 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -24,7 +24,7 @@ def create_root(sub_dir: str, tmp_path: Path) -> Path: (root / "uploads.json").write_bytes(b"{}") (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/C-DRIVE.txt").write_bytes(b"{}") - (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/Test%254Test.evtx").write_bytes(b"{}") + (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/Microsoft-Windows-Windows Defender%254WHC.evtx").write_bytes(b"{}") (root / f"uploads/{sub_dir}/%5C%5C.%5CC%3A/other.txt").write_text("my first file") with open(absolute_path("_data/plugins/filesystem/ntfs/mft/mft.raw"), "rb") as fh: @@ -86,7 +86,7 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat assert paths[5] == target_bare.fs.path("sysvol/other.txt") assert target_bare.fs.path("sysvol/.TEST").exists() - assert target_bare.fs.path("sysvol/Test%4Test.evtx").exists() + assert target_bare.fs.path("Microsoft-Windows-Windows Defender%4WHC.evtx") @pytest.mark.parametrize( @@ -116,7 +116,7 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> assert len(target_bare.filesystems) == 4 assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() assert target_bare.fs.path("sysvol/.TEST").exists() - assert target_bare.fs.path("sysvol/Test%4Test.evtx").exists() + assert target_bare.fs.path("Microsoft-Windows-Windows Defender%4WHC.evtx") @pytest.mark.parametrize( From e3cfc80529e45cd9a68865f60aa7adc1ad1bdcfa Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 20 Nov 2024 09:59:29 +0100 Subject: [PATCH 23/26] Fix --- dissect/target/filesystems/dir.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dissect/target/filesystems/dir.py b/dissect/target/filesystems/dir.py index 5ed2aa023..72e9a2772 100644 --- a/dissect/target/filesystems/dir.py +++ b/dissect/target/filesystems/dir.py @@ -46,8 +46,6 @@ def _resolve_path(self, path: str) -> Path: return entry def get(self, path: str) -> FilesystemEntry: - path = path.strip("/") - if not (path := path.strip("/")): return DirectoryFilesystemEntry(self, "/", self.base_path) From b0e0e14c664176e15352476179975b6258eea039 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 20 Nov 2024 11:02:35 +0100 Subject: [PATCH 24/26] Apply suggestion code review --- dissect/target/loaders/velociraptor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 707c9f365..608a61162 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -138,9 +138,8 @@ def map(self, target: Target) -> None: class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: path = quote(path, safe="$/% ") - path = ( - join(dirname(path), str(basename(path)).replace(".", "%2E", 1)) if basename(path).startswith(".") else path - ) + if (fname := basename(path)).startswith("."): + path = str(join(dirname(path), fname.replace(".", "%2E", 1))) return super()._resolve_path(path) From 15e784d0793fa1ae4b4216a71fd2200a4a034a07 Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Wed, 20 Nov 2024 13:05:12 +0100 Subject: [PATCH 25/26] Remove `str` --- dissect/target/loaders/velociraptor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dissect/target/loaders/velociraptor.py b/dissect/target/loaders/velociraptor.py index 608a61162..a1ead9e0c 100644 --- a/dissect/target/loaders/velociraptor.py +++ b/dissect/target/loaders/velociraptor.py @@ -139,7 +139,8 @@ class VelociraptorDirectoryFilesystem(DirectoryFilesystem): def _resolve_path(self, path: str) -> Path: path = quote(path, safe="$/% ") if (fname := basename(path)).startswith("."): - path = str(join(dirname(path), fname.replace(".", "%2E", 1))) + path = join(dirname(path), fname.replace(".", "%2E", 1)) + return super()._resolve_path(path) From 6493c8e95a027ecbc95ed4c65edc0113fcb1fb3c Mon Sep 17 00:00:00 2001 From: Zawadi Done Date: Tue, 10 Dec 2024 13:23:16 +0100 Subject: [PATCH 26/26] Fix tests `Defender%254WHC.evtx` -> `Defender%4WHC.evtx` --- tests/loaders/test_velociraptor.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/tests/loaders/test_velociraptor.py b/tests/loaders/test_velociraptor.py index 16d184ddb..02cce90d5 100644 --- a/tests/loaders/test_velociraptor.py +++ b/tests/loaders/test_velociraptor.py @@ -76,17 +76,8 @@ def test_windows_ntfs(sub_dir: str, other_dir: str, target_bare: Target, tmp_pat assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() assert target_bare.fs.path("sysvol/other.txt").read_text() == "my first file" - - paths = list(target_bare.fs.path("sysvol").iterdir()) - - assert paths[2].exists() - assert paths[2] == target_bare.fs.path("sysvol/C-DRIVE.txt") - - assert paths[5].exists() - assert paths[5] == target_bare.fs.path("sysvol/other.txt") - assert target_bare.fs.path("sysvol/.TEST").exists() - assert target_bare.fs.path("Microsoft-Windows-Windows Defender%4WHC.evtx") + assert target_bare.fs.path("sysvol/Microsoft-Windows-Windows Defender%254WHC.evtx").exists() @pytest.mark.parametrize( @@ -116,7 +107,7 @@ def test_windows_ntfs_zip(sub_dir: str, target_bare: Target, tmp_path: Path) -> assert len(target_bare.filesystems) == 4 assert target_bare.fs.path("sysvol/C-DRIVE.txt").exists() assert target_bare.fs.path("sysvol/.TEST").exists() - assert target_bare.fs.path("Microsoft-Windows-Windows Defender%4WHC.evtx") + assert target_bare.fs.path("sysvol/Microsoft-Windows-Windows Defender%4WHC.evtx").exists() @pytest.mark.parametrize(