Update grr-api-client version to 3.4.7 and GRR osquery flow/hunt coll…

…ectors (#889) * Update grr-api-client version to 3.4.7 * Update poetry.lock * Update containers * Update poetry.lock * Update poetry.lock * Fix lint error * Fix typo * Trim trailing whitespace * Add file_collection_columns attribute to container * Update osquery collector * Mypy fix * Add grr_hunt * Fix trailing whitespace * Update grr_hunt_osquery recipe * Update doc * Update grr_osquery_flow recipe * Update grr_hosts * Fixes * fix * mypy fixes * Update recipe arg * Update dftimewolf/lib/containers/containers.py Co-authored-by: Ramo <[email protected]> --------- Co-authored-by: Ramo <[email protected]>
log2timeline · Jul 2, 2024 · 0bbb055 · 0bbb055
1 parent dc87e2f
commit 0bbb055
Show file tree

Hide file tree

Showing 12 changed files with 258 additions and 70 deletions.
diff --git a/data/recipes/grr_hunt_osquery.json b/data/recipes/grr_hunt_osquery.json
@@ -7,7 +7,11 @@
         "name": "OsqueryCollector",
         "args": {
             "query": "@osquery_query",
-            "paths": "@osquery_paths"
+            "paths": "@osquery_paths",
+            "remote_configuration_path": "@remote_configuration_path",
+            "local_configuration_path": "@local_configuration_path",
+            "configuration_content": "@configuration_content",
+            "file_collection_columns": "@file_collection_columns"
         }
     },{
         "wants": ["OsqueryCollector"],
@@ -30,6 +34,10 @@
         ["reason", "Reason for collection.", null],
         ["--osquery_query", "Osquery query to hunt for.", null],
         ["--osquery_paths", "Path(s) to text file containing one osquery query per line.", null],
+        ["--remote_configuration_path", "Path to a remote osquery configuration file on the GRR client.", ""],
+        ["--local_configuration_path", "Path to a local osquery configuration file.", ""],
+        ["--configuration_content", "Osquery configuration as a JSON string.", ""],
+        ["--file_collection_columns", "The file collection columns.", ""],
         ["--timeout_millis", "Osquery timeout in milliseconds", 300000, {"format": "regex", "regex": "^\\d+$"}],
         ["--ignore_stderr_errors", "Ignore osquery stderr errors", false],
         ["--approvers", "Emails for GRR approval request.", null],

diff --git a/data/recipes/grr_osquery_flow.json b/data/recipes/grr_osquery_flow.json
@@ -7,7 +7,11 @@
         "name": "OsqueryCollector",
         "args": {
             "query": "@osquery_query",
-            "paths": "@osquery_paths"
+            "paths": "@osquery_paths",
+            "remote_configuration_path": "@remote_configuration_path",
+            "local_configuration_path": "@local_configuration_path",
+            "configuration_content": "@configuration_content",
+            "file_collection_columns": "@file_collection_columns"
         }
     },{
         "wants": ["OsqueryCollector"],
@@ -31,6 +35,10 @@
         ["hostnames", "Hostname(s) to collect the osquery flow from.", null, {"format": "grr_host", "comma_separated": true}],
         ["--osquery_query", "Osquery query to hunt for.", null],
         ["--osquery_paths", "Path(s) to text file containing one osquery query per line.", null],
+        ["--remote_configuration_path", "Path to a remote osquery configuration file on the GRR client.", ""],
+        ["--local_configuration_path", "Path to a local osquery configuration file.", ""],
+        ["--configuration_content", "Osquery configuration as a JSON string.", ""],
+        ["--file_collection_columns", "The file collection columns.", ""],
         ["--timeout_millis", "Osquery timeout in milliseconds", 300000, {"format": "regex", "regex": "^\\d+$"}],
         ["--ignore_stderr_errors", "Ignore osquery stderr errors", false],
         ["--directory", "Directory in which to export results.", null],

diff --git a/dftimewolf/lib/collectors/grr_hosts.py b/dftimewolf/lib/collectors/grr_hosts.py
@@ -1154,13 +1154,17 @@ def _ProcessQuery(
       client: the GRR Client.
       osquery_container: the OSQuery.
     """
-    hunt_args = osquery_flows.OsqueryFlowArgs(
-        query=osquery_container.query,
-        timeout_millis=self.timeout_millis,
-        ignore_stderr_errors=self.ignore_stderr_errors)
+    flow_args = osquery_flows.OsqueryFlowArgs()
+    flow_args.query = osquery_container.query
+    flow_args.timeout_millis = self.timeout_millis
+    flow_args.ignore_stderr_errors = self.ignore_stderr_errors
+    flow_args.configuration_content = osquery_container.configuration_content
+    flow_args.configuration_path = osquery_container.configuration_path
+    flow_args.file_collection_columns.extend(
+        osquery_container.file_collection_columns)
 
     try:
-      flow_id = self._LaunchFlow(client, 'OsqueryFlow', hunt_args)
+      flow_id = self._LaunchFlow(client, 'OsqueryFlow', flow_args)
       self._AwaitFlow(client, flow_id)
     except DFTimewolfError as error:
       self.ModuleError(

diff --git a/dftimewolf/lib/collectors/grr_hunt.py b/dftimewolf/lib/collectors/grr_hunt.py
@@ -401,6 +401,10 @@ def Process(self) -> None:
       hunt_args.query = osquery_container.query
       hunt_args.timeout_millis = self.timeout_millis
       hunt_args.ignore_stderr_errors = self.ignore_stderr_errors
+      hunt_args.configuration_content = osquery_container.configuration_content
+      hunt_args.configuration_path = osquery_container.configuration_path
+      hunt_args.file_collection_columns.extend(
+          osquery_container.file_collection_columns)
 
       self._CreateAndStartHunt('OsqueryFlow', hunt_args)
 

diff --git a/dftimewolf/lib/collectors/osquery.py b/dftimewolf/lib/collectors/osquery.py
@@ -12,11 +12,17 @@
 
 _ALL_PLATFORMS = ['darwin', 'freebsd', 'linux', 'windows']
 
+
 class OsqueryCollector(module.BaseModule):
-  """Osquey query collector.
+  """Osquery collector that creates OsqueryQuery containers.
 
   Attributes:
-      osqueries (List[containers.OsqueryQuery]): list of osquery containers.
+    osqueries (List[containers.OsqueryQuery]): list of osquery containers.
+    configuration_path (str): the path to a configuration file on the
+        client.
+    configuration_content (str): the JSON configuration content.
+    file_collection_columns (List[str]): The list of file collection
+        columns.
   """
 
   def __init__(self,
@@ -34,6 +40,9 @@ def __init__(self,
     super(OsqueryCollector, self).__init__(
         state, name=name, critical=critical)
     self.osqueries: List[containers.OsqueryQuery] = []
+    self.configuration_path: str = ''
+    self.configuration_content: str = ''
+    self.file_collection_columns: List[str] = []
 
   def _ValidateOsquery(self, query: str) -> bool:
     """Validate Osquery query.
@@ -89,24 +98,27 @@ def _LoadOsqueryPackToState(self, path: str) -> None:
     if 'platform' in query_pack:
       global_platform = self._ParsePlatforms(query_pack.get('platform'))
 
-    for num, (name, entry) in enumerate(
-        query_pack.get('queries', {}).items()):
+    for num, (name, entry) in enumerate(query_pack.get('queries', {}).items()):
       query = entry['query']
       if not self._ValidateOsquery(query):
-        self.logger.warning(f'Entry {num} in query pack'
-                            f'{path} does not appear to be valid.')
+        self.logger.warning(
+            f'Entry {num} in query pack {path} does not appear to be valid.')
         continue
 
       if 'platform' in entry:
         platform = self._ParsePlatforms(entry.get('platform'))
       else:
         platform = global_platform
+
       self.osqueries.append(
           containers.OsqueryQuery(
               query=query,
               name=name,
               description=entry.get('description', ''),
-              platforms=platform))
+              platforms=platform,
+              configuration_content=self.configuration_content,
+              configuration_path=self.configuration_path,
+              file_collection_columns=self.file_collection_columns))
 
   def _LoadTextFileToState(self, path: str) -> None:
     """Loads osquery from a text file and creates Osquery containers.
@@ -122,32 +134,89 @@ def _LoadTextFileToState(self, path: str) -> None:
                   query=line,
                   name='',
                   description='',
-                  platforms=None))
+                  platforms=None,
+                  configuration_content=self.configuration_content,
+                  configuration_path=self.configuration_path,
+                  file_collection_columns=self.file_collection_columns))
         else:
           self.logger.warning(f'Osquery on line {line_number} of {path} '
                               'does not appear to be valid.')
 
   # pylint: disable=arguments-differ
-  def SetUp(self,
-            query: str,
-            paths: str) -> None:
+  def SetUp(
+      self,
+      query: str,
+      paths: str,
+      remote_configuration_path: str = '',
+      local_configuration_path: str = '',
+      configuration_content: str = '',
+      file_collection_columns: Optional[str] = None
+  ) -> None:
     """Sets up the osquery to collect.
 
     Supported files are:
     * text files that contain one Osquery
     * json files containing an osquery pack. See https://osquery.readthedocs.io
-          /en/stable/deployment/configuration/#query-packs for details and
-          https://github.com/osquery/osquery/tree/master/packs for examples.
+        /en/stable/deployment/configuration/#query-packs for details and
+        https://github.com/osquery/osquery/tree/master/packs for examples.
+
+    The GRR osquery flow can also be set up to use a custom osquery
+    configuration on invocation (see
+    https://osquery.readthedocs.io/en/stable/deployment/configuration/)
+    either:
+    * as an existing file on the GRR client using remote_configuration_path
+    * as a temporary file on the GRR client where the content can come from
+    a file, using local_cofiguration_path, on the user's local machine or a 
+    string value, using configuration_content.
+    
+    GRR can also collect files based on the results of an Osquery flow using the
+    file_collection_columns argument.
 
     Args:
-      query (str): osquery query.
-      paths (str): osquery filepaths.
+      query: osquery query.
+      paths: osquery filepaths.
+      remote_configuration_path: the path to a remote osquery configuration file
+          on the GRR client.
+      configuration_content: the configuration content, in JSON format.
+      local_configuration_path: the path to a local osquery configuration file.
+      file_collection_columns: The comma-seaparated list of file collection
+          columns names.
     """
     if not query and not paths:
       self.ModuleError('Both query and paths cannot be empty.', critical=True)
 
+    if (remote_configuration_path and (
+        local_configuration_path or configuration_content) or (
+            local_configuration_path and configuration_content
+        )):
+      self.ModuleError(
+          'Only one configuration argument can be set.', critical=True)
+
+    if remote_configuration_path:
+      self.configuration_path = remote_configuration_path
+    elif local_configuration_path:
+      with open(local_configuration_path, mode='r') as fd:
+        configuration_content = fd.read()
+
+    if configuration_content:
+      try:
+        content = json.loads(configuration_content)
+      except json.JSONDecodeError:
+        self.ModuleError(
+            'Osquery configuration does not contain valid JSON.',
+            critical=True)
+      self.configuration_content = json.dumps(content)
+
+    if file_collection_columns:
+      self.file_collection_columns = [
+          col.strip() for col in file_collection_columns.split(',')]
+
     if query and self._ValidateOsquery(query):
-      self.osqueries.append(containers.OsqueryQuery(query=query))
+      self.osqueries.append(containers.OsqueryQuery(
+          query=query,
+          configuration_content=self.configuration_content,
+          configuration_path=self.configuration_path,
+          file_collection_columns=self.file_collection_columns))
     else:
       self.logger.warning(
           'Osquery parameter not set or does not appear to be valid.')

diff --git a/dftimewolf/lib/containers/containers.py b/dftimewolf/lib/containers/containers.py
@@ -599,26 +599,38 @@ class OsqueryQuery(interface.AttributeContainer):
 
   Attributes:
     query (str): The osquery query.
+    configuration_content (str): The JSON content of an osquery
+        configuration.
+    configuration_path (str): The path to an osquery configuration
+        file on the client.
     name (Optional[str]): A name for the osquery.
     platforms (Optional[List[str]]): A constraint on the platform(s) the query
         should be run.  Valid values are 'darwin', 'linux', 'windows',
     description (Optional[str]): A description for the query.
+    file_collection_columns (Optional[List[str]]): The list of file collection
+        columns.
   """
 
   CONTAINER_TYPE = "osquery_query"
 
   def __init__(
     self,
     query: str,
+    configuration_content: str = '',
+    configuration_path: str = '',
     name: Optional[str] = None,
     platforms: Optional[List[str]] = None,
     description: Optional[str] = None,
+    file_collection_columns: Optional[List[str]] = None
   ) -> None:
     super(OsqueryQuery, self).__init__()
     self.description = description
     self.name = name
     self.platforms = platforms
     self.query = query
+    self.configuration_content = configuration_content
+    self.configuration_path = configuration_path
+    self.file_collection_columns = file_collection_columns or []
 
   def __str__(self) -> str:
     """Override __str()__."""

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -27,7 +27,7 @@ boto3 = "^1.24.31"
 python-magic = "^0.4.27"
 turbinia-api-lib = "^1.0.2"
 turbinia-client = "^1.0.3"
-grr-api-client = "^3.4.6"
+grr-api-client = "^3.4.7"
 libcloudforensics = {git = "https://github.com/google/cloud-forensics-utils.git"}
 docker = "^7.0.0"
 setuptools = "^69.5.1"  # needed by docker

diff --git a/tests/lib/collectors/grr_hosts.py b/tests/lib/collectors/grr_hosts.py
@@ -951,7 +951,10 @@ def setUp(self, mock_InitHttp):
     self.grr_osquery_collector = grr_hosts.GRROsqueryCollector(
         self.test_state)
     self.grr_osquery_collector.StoreContainer(
-      containers.OsqueryQuery('SELECT * FROM processes'))
+      containers.OsqueryQuery(
+          'SELECT * FROM processes',
+          configuration_path='/test/path',
+          file_collection_columns=['path']))
     self.grr_osquery_collector.SetUp(
         hostnames='C.0000000000000001',
         reason='Random reason',
@@ -993,7 +996,10 @@ def testProcess(self, mock_LaunchFlow, mock_DownloadResults, _):
         osquery_pb2.OsqueryFlowArgs(
             query='SELECT * FROM processes',
             timeout_millis=300000,
-            ignore_stderr_errors=False
+            ignore_stderr_errors=False,
+            configuration_content='',
+            configuration_path='/test/path',
+            file_collection_columns=['path']
         )
     )