add initial support for $I30 index records

dtformats/data_format.py

@@ -394,6 +394,17 @@ def _FormatString(self, string):
     """
     return string.rstrip('\x00')
 
+  def _FormatByteAsString(self, bytes_object):
+    """Formats a bytes object as a string.
+
+    Args:
+      bytes_object (bytes): bytes object
+
+    Returns:
+      str: a string decoded from bytes
+    """
+    return bytes_object.decode('utf-8')
+
   def _FormatStructureObject(self, structure_object, debug_info):
     """Formats a structure object debug information.
 

dtformats/index_directory_entry.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+"""NTFS $I30 index entries."""
+
+import os
+
+from dtformats import data_format
+from dtformats import errors
+
+
+class NTFSIndexI30Record(data_format.BinaryDataFile):
+  """Class that represents an NTFS $I30 index record."""
+
+  _FABRIC = data_format.BinaryDataFile.ReadDefinitionFile(
+      'index_directory_entry.yml')
+
+  _DEBUG_INDX_ENTRY_HEADER = [
+      ('signature', 'signature', '_FormatByteAsString'),
+      ('fixup_value_offset', 'fixup_value_offset', '_FormatIntegerAsDecimal'),
+      ('num_fixup_values', 'num_fixup_values', '_FormatIntegerAsDecimal'),
+      ('logfile_sequence_number', 'logfile_sequence_number',
+       '_FormatIntegerAsDecimal'),
+      ('virtual_cluster_number', 'virtual_cluster_number',
+       '_FormatIntegerAsDecimal')]
+
+  _DEBUG_INDX_NODE_HEADER = [
+      ('index_values_offset', 'index_values_offset',
+       '_FormatIntegerAsDecimal'),
+      ('index_node_size', 'index_node_size', '_FormatIntegerAsDecimal'),
+      ('allocated_index_node_size', 'allocated_index_node_size',
+       '_FormatIntegerAsDecimal'),
+      ('index_node_flags', 'index_node_flags', '_FormatIntegerAsDecimal')]
+
+  _DEBUG_INDX_DIR_RECORD = [
+      ('file_reference', 'file_reference', '_FormatIntegerAsDecimal'),
+      ('index_value_size', 'index_value_size', '_FormatIntegerAsDecimal'),
+      ('index_key_data_size', 'index_key_data_size', '_FormatIntegerAsDecimal'),
+      ('index_value_flags', 'index_value_flags', '_FormatIntegerAsDecimal')]
+
+  _DEBUG_FILE_NAME_ATTR = [
+      ('parent_file_reference', 'parent_file_reference',
+       '_FormatIntegerAsDecimal'),
+      ('creation_time', 'creation_time', '_FormatIntegerAsDecimal'),
+      ('modification_time', 'modification_time', '_FormatIntegerAsDecimal'),
+      ('entry_modification_time', 'entry_modification_time',
+       '_FormatIntegerAsDecimal'),
+      ('access_time', 'access_time', '_FormatIntegerAsDecimal'),
+      ('allocated_file_size', 'allocated_file_size', '_FormatIntegerAsDecimal'),
+      ('file_size', 'file_size', '_FormatIntegerAsDecimal'),
+      ('file_attribute_flags', 'file_attribute_flags',
+       '_FormatIntegerAsDecimal'),
+      ('extended_data', 'extended_data', '_FormatIntegerAsDecimal'),
+      ('name_size', 'name_size', '_FormatIntegerAsDecimal'),
+      ('name_space', 'name_space', '_FormatIntegerAsDecimal'),
+      ('filename', 'filename', '_FormatString')]
+
+  def PrintRecord(self, record):
+    """Prints a human readable version of the NTFS $I30
+       index record.
+
+    Args:
+        record (index_directory_entry): An index_directory_entry structure.
+    """
+    if record:
+      if self._debug:
+        self._DebugPrintStructureObject(
+            record.entry_header, self._DEBUG_INDX_ENTRY_HEADER)
+        self._DebugPrintStructureObject(
+            record.node_header, self._DEBUG_INDX_NODE_HEADER)
+        self._DebugPrintStructureObject(
+            record, self._DEBUG_INDX_DIR_RECORD)
+        self._DebugPrintStructureObject(
+            record.index_key_data, self._DEBUG_FILE_NAME_ATTR)
+
+  def _ParseIndexEntryHeader(self, file_object):
+    """Parses an NTFS index entry header.
+
+    Args:
+        file_object: A file-like object.
+
+    Returns:
+        tuple[index_entry_header, integer]: A tuple containing an
+            index entry header structure and data size.
+    """
+    file_offset = file_object.tell()
+    data_type_map = self._GetDataTypeMap('index_entry_header')
+
+    index_entry_header, data_size = self._ReadStructureFromFileObject(
+        file_object, file_offset, data_type_map, 'INDX Entry Header')
+    return index_entry_header, data_size
+
+  def _ParseIndexNodeHeader(self, file_object):
+    """Parses an NTFS index node header.
+
+    Args:
+        file_object: A file-like object.
+
+    Returns:
+        tuple[index_node_header, integer]: A tuple containing an index node
+            header structure and data size.
+    """
+    file_offset = file_object.tell()
+    data_type_map = self._GetDataTypeMap('index_node_header')
+
+    index_node_header, data_size = self._ReadStructureFromFileObject(
+        file_object, file_offset, data_type_map, 'INDX Node Header')
+    return index_node_header, data_size
+
+  def _ParseIndexDirectoryEntry(self, file_object):
+    """Parses an NTFS $I30 index record that contains directory entries
+
+    Args:
+        file_object: A file-like object.
+
+    Returns:
+        tuple[index_directory_entry, integer]: A tuple containing a
+            NTFS $I30 index structure and data size.
+    """
+    file_offset = file_object.tell()
+    data_type_map = self._GetDataTypeMap('index_directory_entry')
+
+    try:
+      index_directory_entry, data_size = self._ReadStructureFromFileObject(
+          file_object, file_offset, data_type_map,
+          '4KB block with possible INDX Directory Entry')
+      return index_directory_entry, data_size
+    except errors.ParseError as exception:
+      if self._debug:
+        print(exception)
+      return None, None
+
+  def ReadFileObject(self, file_object):
+    """Reads a file-like object containing INDX records.
+
+    Args:
+      file_object (file): file-like object.
+
+    Raises:
+      ParseError: if the file cannot be read.
+    """
+    self._file_object = file_object
+
+  def ReadRecords(self):
+    """Reads NTFS $I30 INDX records.
+
+    Yields:
+      index_directory_entry: An NTFS $I30 index record.
+
+    Raises:
+      ParseError: if a record cannot be read.
+    """
+    self._file_object.seek(0, os.SEEK_SET)
+    file_offset = 0
+
+    # NTFS INDX entries allocated in 4096-byte chunks
+    block_size = 4096
+
+    while file_offset < self._file_size:
+      self._file_object.seek(file_offset, os.SEEK_SET)
+      index_directory_entry, _ = self._ParseIndexDirectoryEntry(
+          self._file_object)
+      if index_directory_entry:
+        yield index_directory_entry
+
+      file_offset += block_size

dtformats/index_directory_entry.yml

@@ -0,0 +1,158 @@
+name: ntfs_i30_index
+type: format
+description: NTFS $I30 index record that contains directory entries
+urls: ["https://github.com/libyal/libfsntfs/blob/main/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc#index"]
+metadata:
+  authors: ['Juan Leaniz <[email protected]']
+  year: 2021
+---
+name: byte
+type: integer
+attributes:
+  format: unsigned
+  size: 1
+  units: bytes
+---
+name: uint16
+type: integer
+attributes:
+  format: unsigned
+  size: 2
+  units: bytes
+---
+name: uint32
+type: integer
+attributes:
+  format: unsigned
+  size: 4
+  units: bytes
+---
+name: wchar
+type: character
+attributes:
+  size: 1
+  units: bytes
+---
+name: wchar16
+type: character
+description: 16-bit wide character type
+attributes:
+  size: 2
+  units: bytes
+---
+name: wchar32
+type: character
+description: 32-bit wide character type
+attributes:
+  size: 4
+  units: bytes
+---
+name: int64
+type: integer
+description: 64-bit little-endian signed integer type
+attributes:
+  byte_order: little-endian
+  format: signed
+  size: 8
+  units: bytes
+---
+name: uint64
+type: integer
+description: 64-bit little-endian unsigned integer type
+attributes:
+  byte_order: little-endian
+  format: unsigned
+  size: 8
+  units: bytes
+---
+name: file_name_attribute
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: parent_file_reference
+  data_type: uint64
+- name: creation_time
+  data_type: uint64
+- name: modification_time
+  data_type: uint64
+- name: entry_modification_time
+  data_type: uint64
+- name: access_time
+  data_type: uint64
+- name: allocated_file_size
+  data_type: uint64
+- name: file_size
+  data_type: uint64
+- name: file_attribute_flags
+  data_type: uint32
+- name: extended_data
+  data_type: uint32
+- name: name_size
+  data_type: byte
+- name: name_space
+  data_type: byte
+- name: filename
+  type: string
+  encoding: utf-16-le
+  element_data_type: wchar16
+  number_of_elements: file_name_attribute.name_size
+  elements_terminator: "\x00\x00"
+---
+name: index_entry_header
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: signature
+  type: stream
+  element_data_type: byte
+  number_of_elements: 4
+  value: "INDX"
+- name: fixup_value_offset
+  data_type: uint16
+- name: num_fixup_values
+  data_type: uint16
+- name: logfile_sequence_number
+  data_type: int64
+- name: virtual_cluster_number
+  data_type: int64
+---
+name: index_node_header
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: index_values_offset
+  data_type: uint32
+- name: index_node_size
+  data_type: uint32
+- name: allocated_index_node_size
+  data_type: uint32
+- name: index_node_flags
+  data_type: uint32
+---
+name: index_directory_entry
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: entry_header
+  data_type: index_entry_header
+- name: node_header
+  data_type: index_node_header
+- name: values_offset
+  type: stream
+  element_data_type: byte
+  number_of_elements: index_directory_entry.node_header.index_values_offset - 16
+- name: file_reference
+  data_type: uint64
+- name: index_value_size
+  data_type: uint16
+- name: index_key_data_size
+  data_type: uint16
+- name: index_value_flags
+  data_type: uint32
+- name: index_key_data
+  condition: index_directory_entry.index_key_data_size > 64
+  data_type: file_name_attribute