Merge branch 'master' into master

api_client/python/timesketch_api_client/sigma.py

@@ -182,3 +182,17 @@ def from_text(self, rule_text):
         rule_dict = objects[0]
         for key, value in rule_dict.items():
             self.set_value(key, value)
+
+    def delete(self):
+        """Deletes the Sigma rule from Timesketch."""
+        if not self.get_attribute("id"):
+            logger.warning(
+                "Unable to delete the Sigma rule, it does not appear to be "
+                "saved in the first place."
+            )
+            return False
+
+        resource_url = f"{self.api.api_root}/sigmarules/{self.get_attribute('id')}"
+
+        response = self.api.session.delete(resource_url)
+        return error.check_return_status(response, logger)

contrib/deploy_timesketch.sh

@@ -97,7 +97,8 @@ curl -s $GITHUB_BASE_URL/data/intelligence_tag_metadata.yaml > timesketch/etc/ti
 curl -s $GITHUB_BASE_URL/data/sigma_config.yaml > timesketch/etc/timesketch/sigma_config.yaml
 curl -s $GITHUB_BASE_URL/data/sigma_rule_status.csv > timesketch/etc/timesketch/sigma_rule_status.csv
 curl -s $GITHUB_BASE_URL/data/sigma/rules/lnx_susp_zmap.yml > timesketch/etc/timesketch/sigma/rules/lnx_susp_zmap.yml
-curl -s $GITHUB_BASE_URL/data/plaso_formatters.yaml > timesketch/etc/plaso_formatters.yaml
+curl -s $GITHUB_BASE_URL/data/plaso_formatters.yaml > timesketch/etc/timesketch/plaso_formatters.yaml
+curl -s $GITHUB_BASE_URL/data/context_links.yaml > timesketch/etc/timesketch/context_links.yaml
 curl -s $GITHUB_BASE_URL/contrib/nginx.conf > timesketch/etc/nginx.conf
 echo "OK"
 

data/context_links.yaml

@@ -3,8 +3,32 @@
 # ------------------------------------------------------------------------
 #
 # This is a config file to define context links for event attributes.
+# Documentation: https://timesketch.org/guides/admin/context-links/
 #
-# Each context link consists of the following fields:
+# There are two types of context links:
+#
+# 1. Hardcoded modules: These are modules that are hardcoded into Timesketch.
+# The config is used to define the match fields for the module.
+#
+#   module_name:
+#
+#       match_fields:       Type: list[str] | List of field keys where
+#                           this context link should be available. Will
+#                           be checked as case insensitive!
+#
+#       validation_regex:   Type: str | OPTIONAL
+#                           A regex pattern that needs to be
+#                           matched by the field value to to make the
+#                           context link available. This can be used to
+#                           validate the format of a value (e.g. a hash).
+#
+# Currnetly supported modules are:
+#   - XML formatter: Displays a formatted XML in a pop-up dialog.
+#   - Unfurl graph: Displays a graph of an URL using unfurl results.
+#
+# 2. External services: These are context links that are defined by each admin.
+# Those links use external services to provide additional information about the
+# attribute value.
 #
 #   context_link_name:
 #
@@ -37,10 +61,28 @@
 #                                    pages.)
 #
 # ------------------------------------------------------------------------
-## Virustotal Example:
-# virustotal_hash_lookup:
-#   short_name: 'VirusTotal'
-#   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5']
-#   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
-#   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
-#   redirect_warning: TRUE
+## Hardcoded Modules
+hardcoded_modules:
+### format xml dialog
+  xml_formatter:
+    short_name: 'Prettify XML'
+    match_fields:
+      - xml
+      - xml_string
+### unfurl dialog
+  unfurl_graph:
+    short_name: 'Unfurl URL'
+    match_fields:
+      - url
+      - uri
+      - original_url
+
+## External Services
+linked_services:
+### Virustotal Example:
+  # virustotal_hash_lookup:
+  #   short_name: 'VirusTotal'
+  #   match_fields: ['hash', 'sha256_hash', 'sha256', 'sha1_hash', 'sha1', 'md5_hash', 'md5', 'url']
+  #   validation_regex: '/^[0-9a-f]{64}$|^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
+  #   context_link: 'https://www.virustotal.com/gui/search/<ATTR_VALUE>'
+  #   redirect_warning: TRUE

data/features.yaml

@@ -121,149 +121,145 @@ ssh_client_ipv4_addresses:
         query_string: 'reporter:"sshd"'
         attribute: 'message'
         store_as: 'client_ip'
-        re: '^\[sshd\] \[\d+\]: Connection from ((?:[0-9]{1,3}\.){3}[0-9]{1,3})
-          port \d+ on (?:[0-9]{1,3}\.){3}[0-9]{1,3} port \d+(?: rdomain ? .*)?$'
+        re: 'Connection from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+ on (?:[0-9]{1,3}\.){3}[0-9]{1,3} port \d+(?: rdomain ? .*)?$'
 
 ssh_client_ipv4_addresses_2:
         query_string: 'reporter:"sshd"'
         attribute: 'message'
         store_as: 'client_ip'
-        re: '\[sshd, pid: \d+\] Connection [a-z]+ by
-          ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
+        re: 'Connection [a-z]+ by ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
 
 ssh_host_ipv4_addresses:
         query_string: 'reporter:"sshd"'
         attribute: 'message'
         store_as: 'host_ip'
-        re: '^\[sshd\] \[\d+\]: Connection from (?:[0-9]{1,3}\.){3}[0-9]{1,3}
-          port \d+ on ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+(?: rdomain ? .*)?$'
+        re: '^\[sshd\] \[\d+\]: Connection from (?:[0-9]{1,3}\.){3}[0-9]{1,3} port \d+ on ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+(?: rdomain ? .*)?$'
 
 ssh_client_password_ipv4_addresses:
         query_string: 'reporter:"sshd"'
         attribute: 'message'
         store_as: 'client_ip'
-        re: '^\[sshd, pid: \d+\] (?:Accepted|Failed) (?:password|publickey) for \w+
-          from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
+        re: '(?:Accepted|Failed) (?:password|publickey) for \w+ from ((?:[0-9]{1,3}\.){3}[0-9]{1,3}) port \d+'
 
 ssh_disconnected_username:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'username'
-        re: '^Disconnected\s+from user (?P<username>[^\s]+) [^\s]+ port \d+$'
+        re: 'Disconnected\s+from user (?P<username>[^\s]+) [^\s]+ port \d+$'
 
 ssh_disconnected_ip_address:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'ip_address'
-        re: '^Disconnected from user [^\s]+ (?P<ip_address>[^\s]+) port \d+$'
+        re: 'Disconnected from user [^\s]+ (?P<ip_address>[^\s]+) port \d+$'
 
 ssh_disconnected_port:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'port'
-        re: '^Disconnected from user [^\s]+ [^\s]+ port (?P<port>\d+)$'
+        re: 'Disconnected from user [^\s]+ [^\s]+ port (?P<port>\d+)$'
 
 ssh_failed_username:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'username'
-        re: '^Failed password for (?:invalid user)?\s*(?P<username>[^\s]+) from [^\s]+ port \d+ ssh\d'
+        re: 'Failed password for (?:invalid user)?\s*(?P<username>[^\s]+) from [^\s]+ port \d+ ssh\d'
 
 ssh_failed_ip_address:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'ip_address'
-        re: '^Failed password for (?:invalid user)?\s*[^\s]+ from (?P<ip_address>[^\s]+) port \d+ ssh\d'
+        re: 'Failed password for (?:invalid user)?\s*[^\s]+ from (?P<ip_address>[^\s]+) port \d+ ssh\d'
 
 ssh_failed_port:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'port'
-        re: '^Failed password for (?:invalid user)?\s*[^\s]+ from [^\s]+ port (?P<port>\d+) ssh\d'
+        re: 'Failed password for (?:invalid user)?\s*[^\s]+ from [^\s]+ port (?P<port>\d+) ssh\d'
 
 ssh_failed_method:
         query_string: 'reporter:"sshd"'
         attribute: 'body'
         store_as: 'authentication_method'
-        re: '^Failed (?P<authentication_method>[^\s]+) for .*ssh\d'
+        re: 'Failed (?P<authentication_method>[^\s]+) for .*ssh\d'
 
 win_login_subject_username:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'subject_username'
-        re: '.*"SubjectUserName">(?P<subject_username>[^<]+)</Data>'
+        re: '"SubjectUserName">(?P<subject_username>[^<]+)</Data>'
 
 win_login_subject_domain:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'subject_domain'
-        re: '.*"SubjectDomainName">(?P<subject_domain>[^<]+)</Data>'
+        re: '"SubjectDomainName">(?P<subject_domain>[^<]+)</Data>'
 
 win_login_subject_logon_id:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'subject_logon_id'
-        re: '.*"SubjectLogonId">(?P<subject_logon_id>[^<]+)</Data>'
+        re: '"SubjectLogonId">(?P<subject_logon_id>[^<]+)</Data>'
 
 win_login_username:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'username'
-        re: '.*"TargetUserName">(?P<username>[^<]+)</Data>'
+        re: '"TargetUserName">(?P<username>[^<]+)</Data>'
 
 win_login_domain:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'domain'
-        re: '.*"TargetDomainName">(?P<domain>[^<]+)</Data>'
+        re: '"TargetDomainName">(?P<domain>[^<]+)</Data>'
 
 win_login_logon_id:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'logon_id'
-        re: '.*"TargetLogonId">(?P<logon_id>[^<]+)</Data>'
+        re: '"TargetLogonId">(?P<logon_id>[^<]+)</Data>'
 
 win_login_logon_type:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'logon_type'
-        re: '.*"LogonType">(?P<logon_type>[^<]+)</Data>'
+        re: '"LogonType">(?P<logon_type>[^<]+)</Data>'
 
 win_login_logon_process_name:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'logon_process_name'
-        re: '.*"LogonProcessName">(?P<logon_process_name>[^<]+)</Data>'
+        re: '"LogonProcessName">(?P<logon_process_name>[^<]+)</Data>'
 
 win_login_workstation_name:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'workstation_name'
-        re: '.*"WorkstationName">(?P<workstation_name>[^<]+)</Data>'
+        re: '"WorkstationName">(?P<workstation_name>[^<]+)</Data>'
 
 win_login_process_id:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'process_id'
-        re: '.*"ProcessId">(?P<process_id>[^<]+)</Data>'
+        re: '"ProcessId">(?P<process_id>[^<]+)</Data>'
 
 win_login_process_name:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'process_name'
-        re: '.*"ProcessName">(?P<process_name>[^<]+)</Data>'
+        re: '"ProcessName">(?P<process_name>[^<]+)</Data>'
 
 win_login_ip_address:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'ip_address'
-        re: '.*"IpAddress">(?P<ip_address>[^<]+)</Data>'
+        re: '"IpAddress">(?P<ip_address>[^<]+)</Data>'
 
 win_login_port:
         query_string: 'source_name:Microsoft-Windows-Security-Auditing AND (event_identifier:4624 OR event_identifier:4625)'
         attribute: 'xml_string'
         store_as: 'port'
-        re: '.*"IpPort">(?P<port>[^<]+)</Data>'
+        re: '"IpPort">(?P<port>[^<]+)</Data>'
 
 win_bits_client_ipv4_addresses:
         query_string: 'data_type:"windows:evtx:record" AND source_name:Microsoft-Windows-Bits-Client'

docs/developers/api-client.md

@@ -46,6 +46,17 @@ ts_client = config.get_client(token_password='MY_SUPER_L337_PWD')
 
 If the token file does not exist, it will be generated and encrypted using the supplied password.
 
+### Directly passing username / password
+
+Another option to create a connection to the Timesketch server is by creating an `TimesketchApi` object and passing `
+
+```python
+from timesketch_api_client import client as timesketch_client
+client = timesketch_client.TimesketchApi(host_uri='https://demo.timesketch.org', username='demo', password='demo')
+```
+
+> Careful with storing credentials in code that you intend to publish or make available to others. 
+
 ## Client Config
 
 In order to make it simpler to connect to the API client a config file

end_to_end_tests/client_test.py

@@ -85,7 +85,7 @@ def test_direct_opensearch(self):
         data_source = data_sources[0]
         self.assertions.assertEqual(data_source.get("context", ""), context)
 
-    def test_create_sigma_rule(self):
+    def test_sigmarule_create(self):
         """Create a Sigma rule in database"""
         MOCK_SIGMA_RULE = """
 title: Suspicious Installation of bbbbbb
@@ -120,7 +120,7 @@ def test_sigmarule_list(self):
 
         self.assertions.assertIn("installation of bbbbbb", rule.description)
 
-    def test_get_sigmarule(self):
+    def test_sigmarule_create_get(self):
         """Client Sigma object tests."""
 
         rule = self.api.create_sigmarule(
@@ -178,6 +178,23 @@ def test_get_sigmarule(self):
         count = len(data_frame)
         self.assertions.assertEqual(count, 1)
 
+    def test_sigmarule_remove(self):
+        """Client Sigma delete tests.
+        The test is called remove to avoid running it before the create test.
+        """
+        rule = self.api.get_sigmarule(rule_uuid="5266a592-b793-11ea-b3de-eeeee")
+        self.assertions.assertGreater(len(rule.attributes), 5)
+        rule.delete()
+
+        rules = self.api.list_sigmarules()
+        self.assertions.assertGreaterEqual(len(rules), 1)
+
+        rule = self.api.get_sigmarule(rule_uuid="5266a592-b793-11ea-b3de-bbbbbb")
+        self.assertions.assertGreater(len(rule.attributes), 5)
+        rule.delete()
+        rules = self.api.list_sigmarules()
+        self.assertions.assertGreaterEqual(len(rules), 0)
+
     def test_add_event_attributes(self):
         """Tests adding attributes to an event."""
         sketch = self.api.create_sketch(name="Add event attributes test")

requirements.txt

@@ -3,6 +3,7 @@ altair==4.1.0
 celery==5.2.7
 cryptography==41.0.4
 datasketch==1.5.0
+dfir-unfurl==20230901
 opensearch-py==2.3.1
 Flask==2.3.2
 flask_bcrypt==1.0.1

test_tools/test_events/mock_context_links.yaml

@@ -1,20 +1,37 @@
-## Mock configuration file for testing the contrext links API endpoint!
-lookupone:
-  short_name: 'LookupOne'
-  match_fields: ['hash']
-  validation_regex: '/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i'
-  context_link: 'https://lookupone.local/q=<ATTR_VALUE>'
-  redirect_warning: TRUE
+# Mock configuration file for testing the context links API endpoint!
+## Hardcoded Modules
+hardcoded_modules:
+  module_one:
+    short_name: "ModuleOne"
+    validation_regex: "/^[0-9a-f]{64}$/i"
+    match_fields:
+      - xml
+      - xml_string
+  module_two:
+    short_name: "ModuleTwo"
+    match_fields:
+      - url
+      - uri
+      - original_url
 
-lookuptwo:
-  short_name: 'LookupTwo'
-  match_fields: ['sha256_hash', 'hash']
-  validation_regex: '/^[0-9a-f]{64}$/i'
-  context_link: 'https://lookuptwo.local/q=<ATTR_VALUE>'
-  redirect_warning: FALSE
+## External Services
+linked_services:
+  lookupone:
+    short_name: "LookupOne"
+    match_fields: ["hash"]
+    validation_regex: "/^[0-9a-f]{40}$|^[0-9a-f]{32}$/i"
+    context_link: "https://lookupone.local/q=<ATTR_VALUE>"
+    redirect_warning: TRUE
 
-lookupthree:
-  short_name: 'LookupThree'
-  match_fields: ['url']
-  context_link: 'https://lookupthree.local/q=<ATTR_VALUE>'
-  redirect_warning: TRUE
+  lookuptwo:
+    short_name: "LookupTwo"
+    match_fields: ["sha256_hash", "hash"]
+    validation_regex: "/^[0-9a-f]{64}$/i"
+    context_link: "https://lookuptwo.local/q=<ATTR_VALUE>"
+    redirect_warning: FALSE
+
+  lookupthree:
+    short_name: "LookupThree"
+    match_fields: ["url"]
+    context_link: "https://lookupthree.local/q=<ATTR_VALUE>"
+    redirect_warning: TRUE