Optimised AFF4 handler dispatch. FileBackedObject is now able to search for

file under a specified PATH variable. Moved and updated the partition scanner.

Author: scudette

docs/benchmark.txt

@@ -0,0 +1,16 @@
+## This is a little note for me to keep track of performance.
+## All tests were made using --workers=3 on a dual core laptop. Tests
+## ran several times until the disk cache was primed.
+
+rm *tdb; python utilities/Tester.py --uploaddir /var/tmp/uploads/testimages/ -f Live
+Ran 1 test in 26.99s
+Serializing 1.206 sec
+
+Ran 1 test in 14.428s
+Volume closed in 1.27537703514
+
+rm *tdb; python utilities/Tester.py --uploaddir /var/tmp/uploads/testimages/ -f YahooMail
+Ran 1 test in 14.179s
+
+
+

src/plugins/AFF4/AFF4.py

@@ -35,6 +35,10 @@
 PYFLAG_NS = "urn:pyflag:"
 PYFLAG_CASE = PYFLAG_NS + "case"
 
+## Make sure the aff4 subsystem can only load files from the upload
+## dir:
+os.environ['AFF4_FILEPATH'] = "%s:%s" % (config.RESULTDIR, config.UPLOADDIR)
+
 ## Ensure that it does not get exported.
 
 ## These are the supported streams
@@ -73,10 +77,10 @@ def display(self, query, result):
 
         loaded_volumes = []
 
-        for f in filenames:
+        for filename in filenames:
             ## Filenames are always specified relative to the upload
             ## directory
-            filename = "file://%s/%s" % (config.UPLOADDIR, f)
+            #filename = "file://%s/%s" % (config.UPLOADDIR, f)
             volumes = aff4.load_volume(filename)
             result.row("%s" % volumes)
             loaded_volumes.extend(volumes)
@@ -144,8 +148,9 @@ class AFF4ResolverTable(FlagFramework.EventHandler):
     def create(self, dbh, case):
         """ Create a new case AFF4 Result file """
         volume = aff4.ZipVolume(None, 'w')
-        filename = "file://%s/%s.aff4" % (config.RESULTDIR, case)
+        filename = CacheManager.AFF4_MANAGER.make_volume_filename(case)
         aff4.oracle.set(volume.urn, aff4.AFF4_STORED, filename)
+        aff4.oracle.set(filename, aff4.AFF4_CONTAINS, volume.urn)
         volume.finish()
         aff4.oracle.cache_return(volume)
 

src/plugins/AFF4/Loader.py

@@ -0,0 +1,41 @@
+""" This module presents lots of GUIs to build AFF4 objects.
+
+This is not really necessary since aff4 tools can be used on the
+command line to do the same thing.
+
+The basic result is that a new AFF4 object is created in the case
+volume
+"""
+import pyflag.Reports as Reports
+import pyflag.CacheManager as CacheManager
+import pyflag.aff4.aff4 as aff4
+import pyflag.FlagFramework as FlagFramework
+import pdb
+
+class LoadVolume(Reports.report):
+    """ Creates a new AFF4 object in the VFS. """
+    name = 'Load Volume'
+    family = 'Load Data'
+
+    def display(self, query, result):
+        result.start_form(query)
+        result.heading("Load Volume")
+        self.render_form(query, result)
+        result.end_form('Submit')
+
+    def render_form(self, query,result):
+        try:
+            if query["__submit__"]:
+                new_fd = CacheManager.AFF4_MANAGER.create_cache_map(
+                    query['case'], FlagFramework.normpath(query['name']))
+
+                for f in query.getarray('files'):
+                    fd = aff4.oracle.open(f)
+                    new_fd.write_from(fd.urn, 0, fd.size)
+
+                new_fd.close()
+                
+        except KeyError:
+            result.fileselector("Select files to load", 'files')
+            result.text("Files will be logically concatenated in the order selected")
+            result.textfield("Name of object in VFS",'name')

src/plugins_old/DiskForensics/FileHandlers/Partitions.py src/plugins/DiskForensics/FileHandlers/Partitions.py

@@ -14,8 +14,10 @@
 class PartitionScanner(Scanner.GenScanFactory):
     """ Detects partitions in the image and creates VFS nodes for them.
     """
-
-    def scan(self, fd, factories, type, mime):
+    default = True
+    group = "Disk Forensics"
+    
+    def scan(self, fd, scanners, type, mime, cookie, scores=None, **args):
         if 'x86 boot sector' in type:
             try:
                 parts = sk.mmls(fd)
@@ -34,7 +36,7 @@ def scan(self, fd, factories, type, mime):
                 names.add(name)
                 ## Add new maps for each partition
                 map = CacheManager.AFF4_MANAGER.create_cache_map(
-                    self.case,
+                    fd.case,
                     "%s/%s" % (fd.urn, name))
 
                 map.write_from(fd.urn, SECTOR_SIZE * part[0],
@@ -43,24 +45,26 @@ def scan(self, fd, factories, type, mime):
                 map.close()
 
                 ## Now we recursively scan each object
-                new_fd = self.fsfd.open(inode_id = map.inode_id)
+                fsfd = FileSystem.DBFS(fd.case)
+                new_fd = fsfd.open(inode_id = map.inode_id)
                 try:
                     fs = sk.skfs(new_fd)
                     fs.close()
-                    dbh = DB.DBO(self.case)
+                    dbh = DB.DBO(fd.case)
                     dbh.insert("type",
                                inode_id = map.inode_id,
                                mime = "application/filesystem",
                                type = "Filesystem")
                 except: pass
 
-                Scanner.scan_inode(self.case, map.inode_id,
-                                   factories)
+                Scanner.scan_inode_distributed(fd.case, map.inode_id,
+                                               scanners, cookie)
 
 
 class FilesystemLoader(Scanner.GenScanFactory):
     """ A Scanner to automatically load filesystem """
-    def scan(self, fd, factories, type, mime):
+    def scan(self, fd, scanners, type, mime, cookie, scores=None, **args):
+        pdb.set_trace()
         if 'Filesystem' in type:
             print "Will load %s" % fd.urn
 
@@ -80,7 +84,7 @@ def create_map(skfs_inode, path):
                 skfd.seek(0,2)
                 size = skfd.tell()
                 map = CacheManager.AFF4_MANAGER.create_cache_map(
-                    self.case,
+                    fd.case,
                     "%s/__inodes__/%s" % (fd.urn, skfs_inode),
                     size = size,
                     status=status)
@@ -89,7 +93,7 @@ def create_map(skfs_inode, path):
                     map.write_from(fd.urn, block * block_size, block_size)
 
                 CacheManager.AFF4_MANAGER.create_link(
-                    self.case,
+                    fd.case,
                     map.urn, DB.expand("%s/%s",(fd.urn, path)))
                 map.close()
 

src/plugins/Flash/BasicCommands.py

@@ -618,7 +618,8 @@ def execute(self):
 
         m = Magic.MagicResolver()
         for inode_id in self.args:
-            type, mime = m.find_inode_magic(case = self.environment._CASE, inode_id=inode_id)
+            type, mime, scores = m.find_inode_magic(
+                case = self.environment._CASE, inode_id=inode_id)
 
             yield dict(type=type, mime = mime)
 

src/pyflag/CacheManager.py

@@ -154,10 +154,29 @@ def close(self):
 class AFF4Manager:
     """ A Special Cache manager which maintains the main AFF4 Cache
     """
+    def make_volume_filename(self, case):
+        return "%s.aff4" % (case)
+
+    def create_volume(self, case):
+        """ Create a new case AFF4 Result file """
+        volume = aff4.ZipVolume(None, 'w')
+        filename = self.make_volume_filename(case)
+        aff4.oracle.set(volume.urn, aff4.AFF4_STORED, filename)
+        aff4.oracle.set(filename, aff4.AFF4_CONTAINS, volume.urn)
+        volume.finish()
+        aff4.oracle.cache_return(volume)
+
+        return volume.urn
+
     def make_volume_urn(self, case):
-        volume_path = "file://%s/%s.aff4" % (config.RESULTDIR, case)
+        volume_path = self.make_volume_filename(case)
         volume_urn = aff4.oracle.resolve(volume_path, aff4.AFF4_CONTAINS)
 
+        if not volume_urn:
+            ## Volume does not exist - we need to make a new one for
+            ## this case:
+            return self.create_volume(case)
+
         return volume_urn
 
     def close(self, case):

src/pyflag/Scanner.py

@@ -76,7 +76,7 @@ class GenScanFactory:
     ## Relative order of scanners - Higher numbers come later in the order
     order=10
 
-    def scan(self, fd, scanners, type, mime, cookie):
+    def scan(self, fd, scanners, type, mime, cookie, scores=None):
         """ This is the new scan method - scanners just implement this
         method and go from there.