From 71c398a1f64121de8339a51669746ae3e9cdc1a3 Mon Sep 17 00:00:00 2001 From: Brett Fitzpatrick Date: Fri, 27 Sep 2024 05:44:51 -0400 Subject: [PATCH] [Alienvault, CrowdStrike, Phishunt, ThreatFox, URLHaus] added the ability to set x_opencti_score for select connectors (#2554) --- external-import/alienvault/README.md | 42 ++++++---- external-import/alienvault/docker-compose.yml | 10 +++ .../alienvault/src/alienvault/builder.py | 46 ++++++++++- .../alienvault/src/alienvault/core.py | 80 +++++++++++++++++++ .../alienvault/src/alienvault/importer.py | 30 ++++++- .../src/alienvault/utils/__init__.py | 5 +- .../src/alienvault/utils/constants.py | 2 - .../src/alienvault/utils/observables.py | 44 +++------- .../alienvault/src/config.yml.sample | 9 +++ external-import/crowdstrike/README.md | 44 +++++----- .../crowdstrike/docker-compose.yml | 5 ++ .../crowdstrike/src/config.yml.sample | 5 ++ .../src/crowdstrike_feeds_connector/core.py | 34 ++++++++ .../indicator/builder.py | 25 ++++-- .../indicator/importer.py | 15 ++++ .../utils/__init__.py | 7 +- .../utils/config_variables.py | 33 ++++++++ external-import/phishunt/docker-compose.yml | 4 + .../phishunt/src/config.yml.sample | 4 + external-import/phishunt/src/phishunt.py | 56 ++++++++++--- external-import/threatfox/README.md | 7 +- external-import/threatfox/docker-compose.yml | 5 ++ .../threatfox/src/config.yml.sample | 5 ++ external-import/threatfox/src/main.py | 72 +++++++++++++++++ external-import/urlhaus/docker-compose.yml | 1 + external-import/urlhaus/src/config.yml.sample | 1 + external-import/urlhaus/src/urlhaus.py | 12 ++- 27 files changed, 506 insertions(+), 97 deletions(-) diff --git a/external-import/alienvault/README.md b/external-import/alienvault/README.md index a6031c9b90..0bb6fcb321 100644 --- a/external-import/alienvault/README.md +++ b/external-import/alienvault/README.md @@ -43,20 +43,28 @@ Below are the parameters you'll need to set for running the connector properly: Below are the parameters you'll need to set for AlienVault connector: -| Parameter `AlienVault` | config.yml | Docker environment variable | Default | Mandatory | Description | -|----------------------------------|------------------------------------|-----------------------------------------------|-------------------------------|-----------|--------------------------------------------------------------------------------------------------------------------------------| -| Base Url | `base_url` | `ALIENVAULT_BASE_URL` | `https://otx.alienvault.com` | Yes | The base URL for the OTX DirectConnect API. | -| Api Key | `api_key` | `ALIENVAULT_API_KEY` | `ChangeMe` | No | The OTX Key. | -| TLP | `tlp` | `ALIENVAULT_TLP` | `White` | Yes | The default TLP marking used if the Pulse does not define TLP. | -| Create Observables | `create_observables` | `ALIENVAULT_CREATE_OBSERVABLES` | `True` | No | If true then observables will be created from Pulse indicators and added to the report. | -| Create Indicators | `create_indicators` | `ALIENVAULT_CREATE_INDICATORS` | `True` | No | If true then indicators will be created from Pulse indicators and added to the report. | -| Pulse Start Timestamp | `pulse_start_timestamp` | `ALIENVAULT_PULSE_START_TIMESTAMP` | `2020-05-01T00:00:00` | Yes | The Pulses modified after this timestamp will be imported. Timestamp in ISO 8601 format, UTC. | -| Report Status | `report_status` | `ALIENVAULT_REPORT_STATUS` | `New` | Yes | The status of imported reports in the OpenCTI. | -| Report Type | `report_type` | `ALIENVAULT_REPORT_TYPE` | `threat-report` | No | The type of imported reports in the OpenCTI. | -| Guess Malware | `guess_malware` | `ALIENVAULT_GUESS_MALWARE` | `False` | Yes | The Pulse tags are used to guess (queries malwares in the OpenCTI) malwares related to the given Pulse. | -| Guess CVE | `guess_cve` | `ALIENVAULT_GUESS_CVE` | `False` | Yes | The Pulse tags are used to guess (checks whether tag matches (CVE-\d{4}-\d{4,7})) vulnerabilities. | -| Excluded Pulse Indicator Types | `excluded_pulse_indicator_types` | `ALIENVAULT_EXCLUDED_PULSE_INDICATOR_TYPES` | `FileHash-MD5,FileHash-SHA1` | Yes | The Pulse indicator types that will be excluded from the import. | -| Enable Relationships | `enable_relationships` | `ALIENVAULT_ENABLE_RELATIONSHIPS` | `True` | No | If true then the relationships will be created between SDOs. | -| Enable Attack Patterns Indicates | `enable_attack_patterns_indicates` | `ALIENVAULT_ENABLE_ATTACK_PATTERNS_INDICATES` | `True` | No | If true then the relationships `indicates` will be created between indicators and attack patterns. | -| Filter Indicators | `filter_indicators` | `ALIENVAULT_FILTER_INDICATORS` | `True` | No | This boolean filters out indicators created before the latest pulse datetime, ensuring only recent indicators are processed. | - +| Parameter `AlienVault` | config.yml | Docker environment variable | Default | Mandatory | Description | +|---------------------------------------|------------------------------------|-----------------------------------------------|-------------------------------|-----------|--------------------------------------------------------------------------------------------------------------------------------| +| Base Url | `base_url` | `ALIENVAULT_BASE_URL` | `https://otx.alienvault.com` | Yes | The base URL for the OTX DirectConnect API. | +| Api Key | `api_key` | `ALIENVAULT_API_KEY` | `ChangeMe` | No | The OTX Key. | +| TLP | `tlp` | `ALIENVAULT_TLP` | `White` | Yes | The default TLP marking used if the Pulse does not define TLP. | +| Create Observables | `create_observables` | `ALIENVAULT_CREATE_OBSERVABLES` | `True` | No | If true then observables will be created from Pulse indicators and added to the report. | +| Create Indicators | `create_indicators` | `ALIENVAULT_CREATE_INDICATORS` | `True` | No | If true then indicators will be created from Pulse indicators and added to the report. | +| Pulse Start Timestamp | `pulse_start_timestamp` | `ALIENVAULT_PULSE_START_TIMESTAMP` | `2020-05-01T00:00:00` | Yes | The Pulses modified after this timestamp will be imported. Timestamp in ISO 8601 format, UTC. | +| Report Status | `report_status` | `ALIENVAULT_REPORT_STATUS` | `New` | Yes | The status of imported reports in the OpenCTI. | +| Report Type | `report_type` | `ALIENVAULT_REPORT_TYPE` | `threat-report` | No | The type of imported reports in the OpenCTI. | +| Guess Malware | `guess_malware` | `ALIENVAULT_GUESS_MALWARE` | `False` | Yes | The Pulse tags are used to guess (queries malwares in the OpenCTI) malwares related to the given Pulse. | +| Guess CVE | `guess_cve` | `ALIENVAULT_GUESS_CVE` | `False` | Yes | The Pulse tags are used to guess (checks whether tag matches (CVE-\d{4}-\d{4,7})) vulnerabilities. | +| Excluded Pulse Indicator Types | `excluded_pulse_indicator_types` | `ALIENVAULT_EXCLUDED_PULSE_INDICATOR_TYPES` | `FileHash-MD5,FileHash-SHA1` | Yes | The Pulse indicator types that will be excluded from the import. | +| Enable Relationships | `enable_relationships` | `ALIENVAULT_ENABLE_RELATIONSHIPS` | `True` | No | If true then the relationships will be created between SDOs. | +| Enable Attack Patterns Indicates | `enable_attack_patterns_indicates` | `ALIENVAULT_ENABLE_ATTACK_PATTERNS_INDICATES` | `True` | No | If true then the relationships `indicates` will be created between indicators and attack patterns. | +| Filter Indicators | `filter_indicators` | `ALIENVAULT_FILTER_INDICATORS` | `True` | No | This boolean filters out indicators created before the latest pulse datetime, ensuring only recent indicators are processed. | +| default_x_opencti_score | `default_x_opencti_score` | `ALIENVAULT_DEFAULT_X_OPENCTI_SCORE` | `50` | No | The default x_opencti_score to use for indicators. If a per indicator type score is not set, this is used. | +| x_opencti_score_ip | `x_opencti_score_ip` | `ALIENVAULT_X_OPENCTI_SCORE_IP` | `50` | No | The x_opencti_score to use for IP indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_domain | `x_opencti_score_domain` | `ALIENVAULT_X_OPENCTI_SCORE_DOMAIN` | `50` | No | The x_opencti_score to use for Domain indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_hostname | `x_opencti_score_hostname` | `ALIENVAULT_X_OPENCTI_SCORE_HOSTNAME` | `50` | No | The x_opencti_score to use for Hostname indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_email | `x_opencti_score_email` | `ALIENVAULT_X_OPENCTI_SCORE_EMAIL` | `50` | No | The x_opencti_score to use for Email indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_file | `x_opencti_score_file` | `ALIENVAULT_X_OPENCTI_SCORE_FILE` | `50` | No | The x_opencti_score to use for StixFile indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_url | `x_opencti_score_url` | `ALIENVAULT_X_OPENCTI_SCORE_URL` | `50` | No | The x_opencti_score to use for URL indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_mutex | `x_opencti_score_mutex` | `ALIENVAULT_X_OPENCTI_SCORE_MUTEX` | `50` | No | The x_opencti_score to use for Mutex indicators. If not set, the default value is `default_x_opencti_score`. | +| x_opencti_score_cryptocurrency_wallet | `x_opencti_score_cryptocurrency_wallet` | `ALIENVAULT_X_OPENCTI_SCORE_CRYPTOCURRENCY_WALLET` | `50` | No | The x_opencti_score to use for Cryptocurrency Wallet indicators. If not set, the default value is `default_x_opencti_score`. | diff --git a/external-import/alienvault/docker-compose.yml b/external-import/alienvault/docker-compose.yml index 07272294cf..34dded21a7 100644 --- a/external-import/alienvault/docker-compose.yml +++ b/external-import/alienvault/docker-compose.yml @@ -23,4 +23,14 @@ services: - ALIENVAULT_EXCLUDED_PULSE_INDICATOR_TYPES=FileHash-MD5,FileHash-SHA1 # Excluded Pulse indicator types. - ALIENVAULT_ENABLE_RELATIONSHIPS=true # Enable/Disable relationship creation between SDOs. - ALIENVAULT_ENABLE_ATTACK_PATTERNS_INDICATES=false # Enable/Disable "indicates" relationships between indicators and attack patterns + - ALIENVAULT_INTERVAL_SEC=1800 + - ALIENVAULT_DEFAULT_X_OPENCTI_SCORE=50 + - ALIENVAULT_X_OPENCTI_SCORE_IP=60 + - ALIENVAULT_X_OPENCTI_SCORE_DOMAIN=70 + - ALIENVAULT_X_OPENCTI_SCORE_HOSTNAME=75 + - ALIENVAULT_X_OPENCTI_SCORE_EMAIL=70 + - ALIENVAULT_X_OPENCTI_SCORE_FILE=85 + - ALIENVAULT_X_OPENCTI_SCORE_URL=80 + - ALIENVAULT_X_OPENCTI_SCORE_MUTEX=60 + - ALIENVAULT_X_OPENCTI_SCORE_CRYPTOCURRENCY_WALLET=80 restart: always diff --git a/external-import/alienvault/src/alienvault/builder.py b/external-import/alienvault/src/alienvault/builder.py index 49c2640f14..5dcabbaf6a 100644 --- a/external-import/alienvault/src/alienvault/builder.py +++ b/external-import/alienvault/src/alienvault/builder.py @@ -70,6 +70,15 @@ class PulseBundleBuilderConfig(NamedTuple): excluded_pulse_indicator_types: Set[str] enable_relationships: bool enable_attack_patterns_indicates: bool + x_opencti_score: int + x_opencti_score_ip: int + x_opencti_score_domain: int + x_opencti_score_hostname: int + x_opencti_score_email: int + x_opencti_score_file: int + x_opencti_score_url: int + x_opencti_score_mutex: int + x_opencti_score_cryptocurrency_wallet: int class PulseBundleBuilder: @@ -136,6 +145,31 @@ def __init__( self.excluded_pulse_indicator_types = config.excluded_pulse_indicator_types self.enable_relationships = config.enable_relationships self.enable_attack_patterns_indicates = config.enable_attack_patterns_indicates + self.x_opencti_score = { + "default": config.x_opencti_score, + "IPv4": config.x_opencti_score_ip, + "IPv4-Addr": config.x_opencti_score_ip, + "IPv6": config.x_opencti_score_ip, + "IPv6-Addr": config.x_opencti_score_ip, + "CIDR": config.x_opencti_score_ip, + "domain": config.x_opencti_score_domain, + "Domain-Name": config.x_opencti_score_domain, + "hostname": config.x_opencti_score_hostname, + "Hostname": config.x_opencti_score_hostname, + "email": config.x_opencti_score_email, + "Email-addr": config.x_opencti_score_email, + "FilePath": config.x_opencti_score_file, + "FileHash-MD5": config.x_opencti_score_file, + "FileHash-SHA1": config.x_opencti_score_file, + "FileHash-SHA256": config.x_opencti_score_file, + "StixFile": config.x_opencti_score_file, + "URL": config.x_opencti_score_url, + "URI": config.x_opencti_score_url, + "Url": config.x_opencti_score_url, + "Mutex": config.x_opencti_score_mutex, + "BitcoinAddress": config.x_opencti_score_cryptocurrency_wallet, + "Cryptocurrency-Wallet": config.x_opencti_score_cryptocurrency_wallet, + } def _no_relationships(self) -> bool: return not self.enable_relationships @@ -369,7 +403,9 @@ def _create_observations( if self.create_observables: observable_properties = self._create_observable_properties( - pulse_indicator_value, labels + value=pulse_indicator_value, + labels=labels, + x_opencti_score=self.x_opencti_score.get(pulse_indicator_type), ) observable = factory.create_observable(observable_properties) @@ -430,10 +466,10 @@ def _exclude_pulse_indicator_types_filter( ) def _create_observable_properties( - self, value: str, labels: List[str] + self, value: str, labels: List[str], x_opencti_score: int ) -> ObservableProperties: return ObservableProperties( - value, self.pulse_author, labels, self.object_markings + value, self.pulse_author, labels, self.object_markings, x_opencti_score ) def _create_indicator( @@ -456,6 +492,10 @@ def _create_indicator( labels=labels, confidence=self.confidence_level, object_markings=self.object_markings, + x_opencti_score=( + self.x_opencti_score.get(main_observable_type) + or self.x_opencti_score.get("default") + ), x_opencti_main_observable_type=main_observable_type, ) diff --git a/external-import/alienvault/src/alienvault/core.py b/external-import/alienvault/src/alienvault/core.py index fd5b995460..dd74030873 100644 --- a/external-import/alienvault/src/alienvault/core.py +++ b/external-import/alienvault/src/alienvault/core.py @@ -56,6 +56,18 @@ class AlienVault: "closed": 3, } + _CONFIG_DEFAULT_X_OPENCTI_SCORE = f"{_CONFIG_NAMESPACE}.default_x_opencti_score" + _CONFIG_X_OPENCTI_SCORE_IP = f"{_CONFIG_NAMESPACE}.x_opencti_score_ip" + _CONFIG_X_OPENCTI_SCORE_DOMAIN = f"{_CONFIG_NAMESPACE}.x_opencti_score_domain" + _CONFIG_X_OPENCTI_SCORE_HOSTNAME = f"{_CONFIG_NAMESPACE}.x_opencti_score_hostname" + _CONFIG_X_OPENCTI_SCORE_EMAIL = f"{_CONFIG_NAMESPACE}.x_opencti_score_email" + _CONFIG_X_OPENCTI_SCORE_FILE = f"{_CONFIG_NAMESPACE}.x_opencti_score_file" + _CONFIG_X_OPENCTI_SCORE_URL = f"{_CONFIG_NAMESPACE}.x_opencti_score_url" + _CONFIG_X_OPENCTI_SCORE_MUTEX = f"{_CONFIG_NAMESPACE}.x_opencti_score_mutex" + _CONFIG_X_OPENCTI_SCORE_CRYPTOCURRENCY_WALLET = ( + f"{_CONFIG_NAMESPACE}.x_opencti_score_cryptocurrency_wallet" + ) + _DEFAULT_CREATE_OBSERVABLES = True _DEFAULT_CREATE_INDICATORS = True _DEFAULT_FILTER_INDICATORS = True @@ -63,6 +75,7 @@ class AlienVault: _DEFAULT_ENABLE_RELATIONSHIPS = True _DEFAULT_ENABLE_ATTACK_PATTERNS_INDICATES = True _DEFAULT_INTERVAL_SEC = 1800 + _DEFAULT_DEFAULT_X_OPENCTI_SCORE = 50 _STATE_LAST_RUN = "last_run" @@ -148,6 +161,64 @@ def __init__(self) -> None: else: enable_attack_patterns_indicates = bool(enable_attack_patterns_indicates) + # Set x_opencti_score(s) + default_x_opencti_score = ( + self._get_configuration( + config, self._CONFIG_DEFAULT_X_OPENCTI_SCORE, is_number=True + ) + or self._DEFAULT_DEFAULT_X_OPENCTI_SCORE + ) + x_opencti_score_ip = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_IP, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_domain = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_DOMAIN, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_hostname = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_HOSTNAME, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_email = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_EMAIL, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_file = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_FILE, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_url = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_URL, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_mutex = ( + self._get_configuration( + config, self._CONFIG_X_OPENCTI_SCORE_MUTEX, is_number=True + ) + or default_x_opencti_score + ) + x_opencti_score_cryptocurrency_wallet = ( + self._get_configuration( + config, + self._CONFIG_X_OPENCTI_SCORE_CRYPTOCURRENCY_WALLET, + is_number=True, + ) + or default_x_opencti_score + ) + # Create OpenCTI connector helper self.helper = OpenCTIConnectorHelper(config) @@ -186,6 +257,15 @@ def __init__(self) -> None: filter_indicators=filter_indicators, enable_relationships=enable_relationships, enable_attack_patterns_indicates=enable_attack_patterns_indicates, + default_x_opencti_score=default_x_opencti_score, + x_opencti_score_ip=x_opencti_score_ip, + x_opencti_score_domain=x_opencti_score_domain, + x_opencti_score_hostname=x_opencti_score_hostname, + x_opencti_score_email=x_opencti_score_email, + x_opencti_score_file=x_opencti_score_file, + x_opencti_score_url=x_opencti_score_url, + x_opencti_score_mutex=x_opencti_score_mutex, + x_opencti_score_cryptocurrency_wallet=x_opencti_score_cryptocurrency_wallet, ) self.pulse_importer = PulseImporter(pulse_importer_config) diff --git a/external-import/alienvault/src/alienvault/importer.py b/external-import/alienvault/src/alienvault/importer.py index 085841541c..529ffd08ed 100644 --- a/external-import/alienvault/src/alienvault/importer.py +++ b/external-import/alienvault/src/alienvault/importer.py @@ -31,6 +31,15 @@ class PulseImporterConfig(NamedTuple): filter_indicators: bool enable_relationships: bool enable_attack_patterns_indicates: bool + default_x_opencti_score: int + x_opencti_score_ip: int + x_opencti_score_domain: int + x_opencti_score_hostname: int + x_opencti_score_email: int + x_opencti_score_file: int + x_opencti_score_url: int + x_opencti_score_mutex: int + x_opencti_score_cryptocurrency_wallet: int class PulseImporter: @@ -64,7 +73,17 @@ def __init__( self.excluded_pulse_indicator_types = config.excluded_pulse_indicator_types self.enable_relationships = config.enable_relationships self.enable_attack_patterns_indicates = config.enable_attack_patterns_indicates - + self.default_x_opencti_score = config.default_x_opencti_score + self.x_opencti_score_ip = config.x_opencti_score_ip + self.x_opencti_score_domain = config.x_opencti_score_domain + self.x_opencti_score_hostname = config.x_opencti_score_hostname + self.x_opencti_score_email = config.x_opencti_score_email + self.x_opencti_score_file = config.x_opencti_score_file + self.x_opencti_score_url = config.x_opencti_score_url + self.x_opencti_score_mutex = config.x_opencti_score_mutex + self.x_opencti_score_cryptocurrency_wallet = ( + config.x_opencti_score_cryptocurrency_wallet + ) self.malware_guess_cache: Dict[str, str] = {} self.guess_cve_pattern = re.compile(self._GUESS_CVE_PATTERN, re.IGNORECASE) self.work_id: Optional[str] = None @@ -226,6 +245,15 @@ def _create_pulse_bundle(self, pulse: Pulse) -> Optional[stix2.Bundle]: excluded_pulse_indicator_types=self.excluded_pulse_indicator_types, enable_relationships=self.enable_relationships, enable_attack_patterns_indicates=self.enable_attack_patterns_indicates, + x_opencti_score=self.default_x_opencti_score, + x_opencti_score_ip=self.x_opencti_score_ip, + x_opencti_score_domain=self.x_opencti_score_domain, + x_opencti_score_hostname=self.x_opencti_score_hostname, + x_opencti_score_email=self.x_opencti_score_email, + x_opencti_score_file=self.x_opencti_score_file, + x_opencti_score_url=self.x_opencti_score_url, + x_opencti_score_mutex=self.x_opencti_score_mutex, + x_opencti_score_cryptocurrency_wallet=self.x_opencti_score_cryptocurrency_wallet, ) bundle_builder = PulseBundleBuilder(config) diff --git a/external-import/alienvault/src/alienvault/utils/__init__.py b/external-import/alienvault/src/alienvault/utils/__init__.py index 5bb4be1785..c1e0d59b49 100644 --- a/external-import/alienvault/src/alienvault/utils/__init__.py +++ b/external-import/alienvault/src/alienvault/utils/__init__.py @@ -6,7 +6,6 @@ import stix2 from alienvault.utils.constants import ( - DEFAULT_X_OPENCTI_SCORE, TLP_MARKING_DEFINITION_MAPPING, X_MITRE_ID, X_OPENCTI_LOCATION_TYPE, @@ -184,6 +183,7 @@ def create_external_reference( def create_indicator( pattern: str, pattern_type: str, + x_opencti_score: int, created_by: Optional[stix2.Identity] = None, name: Optional[str] = None, description: Optional[str] = None, @@ -194,12 +194,13 @@ def create_indicator( x_opencti_main_observable_type: Optional[str] = None, ) -> stix2.Indicator: """Create an indicator.""" - custom_properties: Dict[str, Any] = {X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE} + custom_properties: Dict[str, Any] = {} if x_opencti_main_observable_type is not None: custom_properties[X_OPENCTI_MAIN_OBSERVABLE_TYPE] = ( x_opencti_main_observable_type ) + custom_properties[X_OPENCTI_SCORE] = x_opencti_score return stix2.Indicator( id=Indicator.generate_id(pattern), diff --git a/external-import/alienvault/src/alienvault/utils/constants.py b/external-import/alienvault/src/alienvault/utils/constants.py index 5ad341b892..0ad7b50f0e 100644 --- a/external-import/alienvault/src/alienvault/utils/constants.py +++ b/external-import/alienvault/src/alienvault/utils/constants.py @@ -23,5 +23,3 @@ X_OPENCTI_LABELS = "x_opencti_labels" X_OPENCTI_CREATED_BY_REF = "x_opencti_created_by_ref" X_OPENCTI_MAIN_OBSERVABLE_TYPE = "x_opencti_main_observable_type" - -DEFAULT_X_OPENCTI_SCORE = 50 diff --git a/external-import/alienvault/src/alienvault/utils/observables.py b/external-import/alienvault/src/alienvault/utils/observables.py index 613e112964..474c92f2e1 100644 --- a/external-import/alienvault/src/alienvault/utils/observables.py +++ b/external-import/alienvault/src/alienvault/utils/observables.py @@ -4,7 +4,6 @@ from typing import Any, List, Mapping, NamedTuple, Optional from alienvault.utils.constants import ( - DEFAULT_X_OPENCTI_SCORE, X_OPENCTI_CREATED_BY_REF, X_OPENCTI_LABELS, X_OPENCTI_SCORE, @@ -24,15 +23,13 @@ def _get_default_custom_properties( + x_opencti_score: int, created_by: Optional[Identity] = None, labels: Optional[List[str]] = None, ) -> Mapping[str, Any]: # XXX: Causes an unexpected property (x_opencti_score) error # when creating a Bundle without allow_custom=True flag. - custom_properties = { - X_OPENCTI_LABELS: labels, - X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE, - } + custom_properties = {X_OPENCTI_LABELS: labels, X_OPENCTI_SCORE: x_opencti_score} if created_by is not None: custom_properties[X_OPENCTI_CREATED_BY_REF] = created_by["id"] @@ -47,11 +44,14 @@ class ObservableProperties(NamedTuple): created_by: Identity labels: List[str] object_markings: List[MarkingDefinition] + x_opencti_score: int def _get_custom_properties(properties: ObservableProperties) -> Mapping[str, Any]: return _get_default_custom_properties( - created_by=properties.created_by, labels=properties.labels + created_by=properties.created_by, + labels=properties.labels, + x_opencti_score=properties.x_opencti_score, ) @@ -112,60 +112,42 @@ def create_observable_url(properties: ObservableProperties) -> URL: def _create_observable_file( + properties: ObservableProperties, hashes: Optional[Mapping[str, str]] = None, name: Optional[str] = None, - created_by: Optional[Identity] = None, - labels: Optional[List[str]] = None, - object_markings: Optional[List[MarkingDefinition]] = None, ) -> File: return File( hashes=hashes, name=name, - object_marking_refs=object_markings, - custom_properties=_get_default_custom_properties( - created_by=created_by, labels=labels - ), + object_marking_refs=properties.object_markings, + custom_properties=_get_custom_properties(properties), ) def create_observable_file_md5(properties: ObservableProperties) -> File: """Create an observable representing a MD5 hash of a file.""" return _create_observable_file( - hashes={"MD5": properties.value}, - created_by=properties.created_by, - labels=properties.labels, - object_markings=properties.object_markings, + hashes={"MD5": properties.value}, properties=properties ) def create_observable_file_sha1(properties: ObservableProperties) -> File: """Create an observable representing a SHA-1 hash of a file.""" return _create_observable_file( - hashes={"SHA-1": properties.value}, - created_by=properties.created_by, - labels=properties.labels, - object_markings=properties.object_markings, + hashes={"SHA-1": properties.value}, properties=properties ) def create_observable_file_sha256(properties: ObservableProperties) -> File: """Create an observable representing a SHA-256 hash of a file.""" return _create_observable_file( - hashes={"SHA-256": properties.value}, - created_by=properties.created_by, - labels=properties.labels, - object_markings=properties.object_markings, + hashes={"SHA-256": properties.value}, properties=properties ) def create_observable_file_name(properties: ObservableProperties) -> File: """Create an observable representing a file name.""" - return _create_observable_file( - name=properties.value, - created_by=properties.created_by, - labels=properties.labels, - object_markings=properties.object_markings, - ) + return _create_observable_file(name=properties.value, properties=properties) def create_observable_mutex(properties: ObservableProperties) -> Mutex: diff --git a/external-import/alienvault/src/config.yml.sample b/external-import/alienvault/src/config.yml.sample index 83b3e79b2f..33dd1e7913 100644 --- a/external-import/alienvault/src/config.yml.sample +++ b/external-import/alienvault/src/config.yml.sample @@ -25,3 +25,12 @@ alienvault: enable_relationships: true # Enable/Disable relationship creation between SDOs enable_attack_patterns_indicates: false # Enable/Disable "indicates" relationships between indicators and attack patterns filter_indicators: true # Filter indicators by their created datetime + default_x_opencti_score: 50 # The default x_opencti_score to use for indicators. If a per indicator type score is not set, this is used. + x_opencti_score_ip: 60 # (Optional): The x_opencti_score to use for IP indicators. + x_opencti_score_domain: 70 # (Optional): The x_opencti_score to use for Domain indicators + x_opencti_score_hostname: 75 # (Optional): The x_opencti_score to use for Hostname indicators. + x_opencti_score_email: 70 # (Optional): The x_opencti_score to use for Email indicators. + x_opencti_score_file: 80 # (Optional): The x_opencti_score to use for StixFile indicators + x_opencti_score_url: 80 # (Optional): The x_opencti_score to use for URL indicators. + x_opencti_score_mutex: 60 # (Optional): The x_opencti_score to use for Mutex indicators. + x_opencti_score_cryptocurrency_wallet: 80 # (Optional): The x_opencti_score to use for Cryptocurrency Wallet indicators diff --git a/external-import/crowdstrike/README.md b/external-import/crowdstrike/README.md index 844ba9394b..84fc407144 100644 --- a/external-import/crowdstrike/README.md +++ b/external-import/crowdstrike/README.md @@ -46,25 +46,29 @@ Below are the parameters you'll need to set for running the connector properly: Below are the parameters you'll need to set for CrowdStrike Connector: -| Parameter `CrowdStrike` | config.yml | Docker environment variable | Default | Mandatory | Example | Description | -|-----------------------------|------------------------------|------------------------------------------|-------------------------------|------------|------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------| -| Base Url | `base_url` | `CROWDSTRIKE_BASE_URL` | `https://api.crowdstrike.com` | No | / | The base URL for the CrowdStrike APIs. | -| Client ID | `client_id` | `CROWDSTRIKE_CLIENT_ID` | `ChangeMe` | Yes | `ChangeMe` | The CrowdStrike API client ID. | -| Client Secret | `client_secret` | `CROWDSTRIKE_CLIENT_SECRET` | `ChangeMe` | Yes | `ChangeMe` | The CrowdStrike API client secret. | -| TLP | `tlp` | `CROWDSTRIKE_TLP` | `Amber` | No | / | The TLP marking used for the imported objects in the OpenCTI. | -| Create Observables | `create_observables` | `CROWDSTRIKE_CREATE_OBSERVABLES` | / | Yes | `true` | If true then observables will be created from the CrowdStrike indicators. | -| Create Indicators | `create_indicators` | `CROWDSTRIKE_CREATE_INDICATORS` | / | Yes | `true` | If true then indicators will be created from the CrowdStrike indicators. | -| Scopes | `scopes` | `CROWDSTRIKE_SCOPES` | / | Yes | `actor,report,indicator,yara_master,snort_suricata_master` | The scopes defines what data will be imported from the CrowdStrike. | -| Actor Start Timestamp | `actor_start_timestamp` | `CROWDSTRIKE_ACTOR_START_TIMESTAMP` | / | Yes | `0` | The Actors created after this timestamp will be imported. Timestamp in UNIX Epoch time, UTC. | -| Report Start Timestamp | `report_start_timestamp` | `CROWDSTRIKE_REPORT_START_TIMESTAMP` | / | Yes | `0` | The Reports created after this timestamp will be imported. Timestamp in UNIX Epoch time, UTC. | -| Report Status | `report_status` | `CROWDSTRIKE_REPORT_STATUS` | / | Yes | `New` | The status of imported reports in the OpenCTI. | -| Report Include Types | `report_include_types` | `CROWDSTRIKE_REPORT_INCLUDE_TYPES` | / | Yes | `notice,tipper,intelligence report,periodic report` | The types of Reports included in the import. The types are defined by the CrowdStrike. | -| Report Type | `report_type` | `CROWDSTRIKE_REPORT_TYPE` | / | Yes | `Threat Report` | The type of imported reports in the OpenCTI. | -| Report Guess Malware | `report_guess_malware` | `CROWDSTRIKE_REPORT_GUESS_MALWARE` | / | Yes | `false` | The Report tags are used to guess (queries malwares in the OpenCTI) malwares related to the given Report. | -| Indicator Start Timestamp | `indicator_start_timestamp` | `CROWDSTRIKE_INDICATOR_START_TIMESTAMP` | / | Yes | `0` | The Indicators published after this timestamp will be imported. Timestamp in UNIX Epoch time, UTC. | -| Indicator Exclude Types | `indicator_exclude_types` | `CROWDSTRIKE_INDICATOR_EXCLUDE_TYPES` | / | Yes | `hash_ion,hash_md5,hash_sha1` | The types of Indicators excluded from the import. The types are defined by the CrowdStrike. | -| Indicator Low Score | `indicator_low_score` | `CROWDSTRIKE_INDICATOR_LOW_SCORE` | / | Yes | `40` | If any of the low score labels are found on the indicator then this value is used as a score. | -| Indicator Low Score Labels | `indicator_low_score_labels` | `CROWDSTRIKE_INDICATOR_LOW_SCORE_LABELS` | / | Yes | `MaliciousConfidence/Low` or `MaliciousConfidence/Medium` | The labels used to determine the low score indicators. | -| Indicator Unwanted Labels | `indicator_unwanted_labels` | `CROWDSTRIKE_INDICATOR_UNWANTED_LABELS` | / | No | / | Indicators to be excluded from import based on the labels affixed to them. | +| Parameter `CrowdStrike` | config.yml | Docker environment variable | Default | Mandatory | Example | Description | +|-------------------------------|---------------------------------|---------------------------------------------|-------------------------------|------------|------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------| +| Base Url | `base_url` | `CROWDSTRIKE_BASE_URL` | `https://api.crowdstrike.com` | No | / | The base URL for the CrowdStrike APIs. | +| Client ID | `client_id` | `CROWDSTRIKE_CLIENT_ID` | `ChangeMe` | Yes | `ChangeMe` | The CrowdStrike API client ID. | +| Client Secret | `client_secret` | `CROWDSTRIKE_CLIENT_SECRET` | `ChangeMe` | Yes | `ChangeMe` | The CrowdStrike API client secret. | +| TLP | `tlp` | `CROWDSTRIKE_TLP` | `Amber` | No | / | The TLP marking used for the imported objects in the OpenCTI. | +| Create Observables | `create_observables` | `CROWDSTRIKE_CREATE_OBSERVABLES` | / | Yes | `true` | If true then observables will be created from the CrowdStrike indicators. | +| Create Indicators | `create_indicators` | `CROWDSTRIKE_CREATE_INDICATORS` | / | Yes | `true` | If true then indicators will be created from the CrowdStrike indicators. | +| Scopes | `scopes` | `CROWDSTRIKE_SCOPES` | / | Yes | `actor,report,indicator,yara_master,snort_suricata_master` | The scopes defines what data will be imported from the CrowdStrike. | +| Actor Start Timestamp | `actor_start_timestamp` | `CROWDSTRIKE_ACTOR_START_TIMESTAMP` | / | Yes | `0` | The Actors created after this timestamp will be imported. Timestamp in UNIX Epoch time, UTC. | +| Report Start Timestamp | `report_start_timestamp` | `CROWDSTRIKE_REPORT_START_TIMESTAMP` | / | Yes | `0` | The Reports created after this timestamp will be imported. Timestamp in UNIX Epoch time, UTC. | +| Report Status | `report_status` | `CROWDSTRIKE_REPORT_STATUS` | / | Yes | `New` | The status of imported reports in the OpenCTI. | +| Report Include Types | `report_include_types` | `CROWDSTRIKE_REPORT_INCLUDE_TYPES` | / | Yes | `notice,tipper,intelligence report,periodic report` | The types of Reports included in the import. The types are defined by the CrowdStrike. | +| Report Type | `report_type` | `CROWDSTRIKE_REPORT_TYPE` | / | Yes | `threat-report` | The type of imported reports in the OpenCTI. | +| Report Guess Malware | `report_guess_malware` | `CROWDSTRIKE_REPORT_GUESS_MALWARE` | / | Yes | `false` | The Report tags are used to guess (queries malwares in the OpenCTI) malwares related to the given Report. | +| Indicator Start Timestamp | `indicator_start_timestamp` | `CROWDSTRIKE_INDICATOR_START_TIMESTAMP` | / | Yes | `0` | The Indicators published after this timestamp will be imported. Timestamp in UNIX Epoch time, UTC. | +| Indicator Exclude Types | `indicator_exclude_types` | `CROWDSTRIKE_INDICATOR_EXCLUDE_TYPES` | / | Yes | `hash_ion,hash_md5,hash_sha1` | The types of Indicators excluded from the import. The types are defined by the CrowdStrike. | +| Indicator Low Score | `indicator_low_score` | `CROWDSTRIKE_INDICATOR_LOW_SCORE` | / | No | `40` | If any of the low score labels are found on the indicator then this value is used as a score. | +| Indicator Low Score Labels | `indicator_low_score_labels` | `CROWDSTRIKE_INDICATOR_LOW_SCORE_LABELS` | / | No | `MaliciousConfidence/Low` or `MaliciousConfidence/Medium` | The labels used to determine the low score indicators. | +| Indicator Medium Score | `indicator_medium_score` | `CROWDSTRIKE_INDICATOR_MEDIUM_SCORE` | / | No | `60` | If any of the low score labels are found on the indicator then this value is used as a score. | +| Indicator Medium Score Labels | `indicator_medium_score_labels` | `CROWDSTRIKE_INDICATOR_MEDIUM_SCORE_LABELS` | / | No | `MaliciousConfidence/Medium` | The labels used to determine the low score indicators. | +| Indicator High Score | `indicator_high_score` | `CROWDSTRIKE_INDICATOR_HIGH_SCORE` | / | No | `80` | If any of the low score labels are found on the indicator then this value is used as a score. | +| Indicator High Score Labels | `indicator_high_score_labels` | `CROWDSTRIKE_INDICATOR_HIGH_SCORE_LABELS` | / | No | `MaliciousConfidence/High` | The labels used to determine the low score indicators. | +| Indicator Unwanted Labels | `indicator_unwanted_labels` | `CROWDSTRIKE_INDICATOR_UNWANTED_LABELS` | / | No | / | Indicators to be excluded from import based on the labels affixed to them. | **Note**: It is not recommended to use the default value `0` for configuration parameters `report_start_timestamp` and `indicator_start_timestamp` because of the large data volumes. diff --git a/external-import/crowdstrike/docker-compose.yml b/external-import/crowdstrike/docker-compose.yml index ee3c1c878b..5c9cc4c5da 100644 --- a/external-import/crowdstrike/docker-compose.yml +++ b/external-import/crowdstrike/docker-compose.yml @@ -25,7 +25,12 @@ services: - CROWDSTRIKE_REPORT_GUESS_MALWARE=false # Use report tags to guess malware - CROWDSTRIKE_INDICATOR_START_TIMESTAMP=0 # BEWARE! A lot of indicators! - CROWDSTRIKE_INDICATOR_EXCLUDE_TYPES=hash_ion,hash_md5,hash_sha1 + - CROWDSTRIKE_DEFAULT_X_OPENCTI_SCORE=50 - CROWDSTRIKE_INDICATOR_LOW_SCORE=40 - CROWDSTRIKE_INDICATOR_LOW_SCORE_LABELS=MaliciousConfidence/Low + - CROWDSTRIKE_INDICATOR_MEDIUM_SCORE=60 + - CROWDSTRIKE_INDICATOR_MEDIUM_SCORE_LABELS=MaliciousConfidence/Medium + - CROWDSTRIKE_INDICATOR_HIGH_SCORE=80 + - CROWDSTRIKE_INDICATOR_HIGH_SCORE_LABELS=MaliciousConfidence/High - CROWDSTRIKE_INDICATOR_UNWANTED_LABELS= # Can be used to filter low confidence indicators: "MaliciousConfidence/Low", "MaliciousConfidence/Medium" restart: always diff --git a/external-import/crowdstrike/src/config.yml.sample b/external-import/crowdstrike/src/config.yml.sample index c261e75a5b..4081671aaa 100644 --- a/external-import/crowdstrike/src/config.yml.sample +++ b/external-import/crowdstrike/src/config.yml.sample @@ -26,6 +26,11 @@ crowdstrike: report_guess_malware: false # Use report tags to guess malware indicator_start_timestamp: 0 # BEWARE! All the indicators! indicator_exclude_types: 'hash_ion,hash_md5,hash_sha1' + default_x_opencti_score: 50 indicator_low_score: 40 indicator_low_score_labels: 'MaliciousConfidence/Low' + indicator_medium_score: 60 + indicator_medium_score_labels: 'MaliciousConfidence/Medium' + indicator_high_score: 80 + indicator_high_score_labels: 'MaliciousConfidence/High' indicator_unwanted_labels: '' # Can be used to filter low confidence indicators: "MaliciousConfidence/Low", "MaliciousConfidence/Medium" diff --git a/external-import/crowdstrike/src/crowdstrike_feeds_connector/core.py b/external-import/crowdstrike/src/crowdstrike_feeds_connector/core.py index cf2f09c66e..065668d737 100644 --- a/external-import/crowdstrike/src/crowdstrike_feeds_connector/core.py +++ b/external-import/crowdstrike/src/crowdstrike_feeds_connector/core.py @@ -46,7 +46,10 @@ class CrowdStrike: _DEFAULT_CREATE_OBSERVABLES = True _DEFAULT_CREATE_INDICATORS = True _DEFAULT_REPORT_TYPE = "threat-report" + _DEFAULT_X_OPENCTI_SCORE = 50 _DEFAULT_INDICATOR_LOW_SCORE = 40 + _DEFAULT_INDICATOR_MEDIUM_SCORE = 60 + _DEFAULT_INDICATOR_HIGH_SCORE = 80 _STATE_LAST_RUN = "last_run" @@ -115,6 +118,10 @@ def __init__(self) -> None: indicator_exclude_types_str ) + default_x_opencti_score = self.config.default_x_opencti_score + if default_x_opencti_score is None: + default_x_opencti_score = self._DEFAULT_X_OPENCTI_SCORE + indicator_low_score = self.config.indicator_low_score if indicator_low_score is None: indicator_low_score = self._DEFAULT_INDICATOR_LOW_SCORE @@ -126,6 +133,28 @@ def __init__(self) -> None: indicator_low_score_labels_str ) + indicator_medium_score = self.config.indicator_medium_score + if indicator_medium_score is None: + indicator_medium_score = self._DEFAULT_INDICATOR_MEDIUM_SCORE + + indicator_medium_score_labels_str = self.config.indicator_medium_score_labels + indicator_medium_score_labels = [] + if indicator_medium_score_labels_str is not None: + indicator_medium_score_labels = convert_comma_separated_str_to_list( + indicator_medium_score_labels_str + ) + + indicator_high_score = self.config.indicator_high_score + if indicator_high_score is None: + indicator_high_score = self._DEFAULT_INDICATOR_HIGH_SCORE + + indicator_high_score_labels_str = self.config.indicator_high_score_labels + indicator_high_score_labels = [] + if indicator_high_score_labels_str is not None: + indicator_high_score_labels = convert_comma_separated_str_to_list( + indicator_high_score_labels_str + ) + indicator_unwanted_labels_str = self.config.indicator_unwanted_labels indicator_unwanted_labels = [] if indicator_unwanted_labels_str is not None: @@ -179,8 +208,13 @@ def __init__(self) -> None: exclude_types=indicator_exclude_types, report_status=report_status, report_type=report_type, + default_x_opencti_score=default_x_opencti_score, indicator_low_score=indicator_low_score, indicator_low_score_labels=set(indicator_low_score_labels), + indicator_medium_score=indicator_medium_score, + indicator_medium_score_labels=set(indicator_medium_score_labels), + indicator_high_score=indicator_high_score, + indicator_high_score_labels=set(indicator_high_score_labels), indicator_unwanted_labels=set(indicator_unwanted_labels), ) diff --git a/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/builder.py b/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/builder.py index fabf09f75b..850c96bc97 100644 --- a/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/builder.py +++ b/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/builder.py @@ -5,7 +5,6 @@ from typing import List, Mapping, NamedTuple, Optional, Set from crowdstrike_feeds_services.utils import ( - DEFAULT_X_OPENCTI_SCORE, OBSERVATION_FACTORY_CRYPTOCURRENCY_WALLET, OBSERVATION_FACTORY_DOMAIN_NAME, OBSERVATION_FACTORY_EMAIL_ADDRESS, @@ -64,11 +63,16 @@ class IndicatorBundleBuilderConfig(NamedTuple): confidence_level: int create_observables: bool create_indicators: bool + default_x_opencti_score: int indicator_report_status: int indicator_report_type: str indicator_reports: List[FetchedReport] indicator_low_score: int indicator_low_score_labels: Set[str] + indicator_medium_score: int + indicator_medium_score_labels: Set[str] + indicator_high_score: int + indicator_high_score_labels: Set[str] indicator_unwanted_labels: Set[str] @@ -132,11 +136,16 @@ def __init__(self, config: IndicatorBundleBuilderConfig) -> None: self.confidence_level = config.confidence_level self.create_observables = config.create_observables self.create_indicators = config.create_indicators + self.default_x_opencti_score = config.default_x_opencti_score self.indicator_reports = config.indicator_reports self.indicator_report_status = config.indicator_report_status self.indicator_report_type = config.indicator_report_type self.indicator_low_score = config.indicator_low_score self.indicator_low_score_labels = config.indicator_low_score_labels + self.indicator_medium_score = config.indicator_medium_score + self.indicator_medium_score_labels = config.indicator_medium_score_labels + self.indicator_high_score = config.indicator_high_score + self.indicator_high_score_labels = config.indicator_high_score_labels self.indicator_unwanted_labels = config.indicator_unwanted_labels self.observation_factory = self._get_observation_factory(self.indicator["type"]) @@ -337,14 +346,20 @@ def _create_observable_properties( ) def _determine_score_by_labels(self, labels: List[str]) -> int: - score = DEFAULT_X_OPENCTI_SCORE + label_score = None + # Score will be given floored at lowest score label found. for label in labels: if label in self.indicator_low_score_labels: - score = self.indicator_low_score + label_score = self.indicator_low_score break - - return score + if label in self.indicator_medium_score_labels: + if label_score is None or label_score > self.indicator_medium_score: + label_score = self.indicator_medium_score + elif label in self.indicator_high_score_labels: + if label_score is None: + label_score = self.indicator_high_score + return label_score if label_score is not None else self.default_x_opencti_score def _create_indicator( self, diff --git a/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/importer.py b/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/importer.py index 6ab23931e6..c73d9ff9d6 100644 --- a/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/importer.py +++ b/external-import/crowdstrike/src/crowdstrike_feeds_connector/indicator/importer.py @@ -30,8 +30,13 @@ class IndicatorImporterConfig(NamedTuple): exclude_types: List[str] report_status: int report_type: str + default_x_opencti_score: int indicator_low_score: int indicator_low_score_labels: Set[str] + indicator_medium_score: int + indicator_medium_score_labels: Set[str] + indicator_high_score: int + indicator_high_score_labels: Set[str] indicator_unwanted_labels: Set[str] @@ -55,8 +60,13 @@ def __init__(self, config: IndicatorImporterConfig) -> None: self.exclude_types = config.exclude_types self.report_status = config.report_status self.report_type = config.report_type + self.default_x_opencti_score = config.default_x_opencti_score self.indicator_low_score = config.indicator_low_score self.indicator_low_score_labels = config.indicator_low_score_labels + self.indicator_medium_score = config.indicator_medium_score + self.indicator_medium_score_labels = config.indicator_medium_score_labels + self.indicator_high_score = config.indicator_high_score + self.indicator_high_score_labels = config.indicator_high_score_labels self.indicator_unwanted_labels = config.indicator_unwanted_labels self.next_page: Optional[str] = None @@ -229,8 +239,13 @@ def _create_indicator_bundle( indicator_report_status=self.report_status, indicator_report_type=self.report_type, indicator_reports=indicator_reports, + default_x_opencti_score=self.default_x_opencti_score, indicator_low_score=self.indicator_low_score, indicator_low_score_labels=self.indicator_low_score_labels, + indicator_medium_score=self.indicator_medium_score, + indicator_medium_score_labels=self.indicator_medium_score_labels, + indicator_high_score=self.indicator_high_score, + indicator_high_score_labels=self.indicator_high_score_labels, indicator_unwanted_labels=self.indicator_unwanted_labels, ) diff --git a/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/__init__.py b/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/__init__.py index 4ca933329b..edd727a17c 100644 --- a/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/__init__.py +++ b/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/__init__.py @@ -982,10 +982,11 @@ def create_indicator( x_opencti_score: Optional[int] = None, ) -> stix2.Indicator: """Create an indicator.""" - custom_properties: Dict[str, Any] = {X_OPENCTI_SCORE: DEFAULT_X_OPENCTI_SCORE} + custom_properties: Dict[str, Any] = {} - if x_opencti_score is not None: - custom_properties[X_OPENCTI_SCORE] = x_opencti_score + custom_properties[X_OPENCTI_SCORE] = ( + x_opencti_score if x_opencti_score is not None else DEFAULT_X_OPENCTI_SCORE + ) if x_opencti_main_observable_type is not None: custom_properties[X_OPENCTI_MAIN_OBSERVABLE_TYPE] = ( diff --git a/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/config_variables.py b/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/config_variables.py index 7fda9ddc25..f092e8d515 100644 --- a/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/config_variables.py +++ b/external-import/crowdstrike/src/crowdstrike_feeds_services/utils/config_variables.py @@ -142,6 +142,13 @@ def _initialize_configurations(self) -> None: self.load, ) + self.default_x_opencti_score: int = get_config_variable( + "CROWDSTRIKE_DEFAULT_X_OPENCTI_SCORE", + ["crowdstrike", "default_x_opencti_score"], + self.load, + isNumber=True, + ) + self.indicator_low_score: int = get_config_variable( "CROWDSTRIKE_INDICATOR_LOW_SCORE", ["crowdstrike", "indicator_low_score"], @@ -155,6 +162,32 @@ def _initialize_configurations(self) -> None: self.load, ) + self.indicator_medium_score: int = get_config_variable( + "CROWDSTRIKE_INDICATOR_MEDIUM_SCORE", + ["crowdstrike", "indicator_medium_score"], + self.load, + isNumber=True, + ) + + self.indicator_medium_score_labels: str = get_config_variable( + "CROWDSTRIKE_INDICATOR_MEDIUM_SCORE_LABELS", + ["crowdstrike", "indicator_medium_score_labels"], + self.load, + ) + + self.indicator_high_score: int = get_config_variable( + "CROWDSTRIKE_INDICATOR_HIGH_SCORE", + ["crowdstrike", "indicator_high_score"], + self.load, + isNumber=True, + ) + + self.indicator_high_score_labels: str = get_config_variable( + "CROWDSTRIKE_INDICATOR_HIGH_SCORE_LABELS", + ["crowdstrike", "indicator_high_score_labels"], + self.load, + ) + self.indicator_unwanted_labels: str = get_config_variable( "CROWDSTRIKE_INDICATOR_UNWANTED_LABELS", ["crowdstrike", "indicator_unwanted_labels"], diff --git a/external-import/phishunt/docker-compose.yml b/external-import/phishunt/docker-compose.yml index dd562eded5..00405b85d4 100644 --- a/external-import/phishunt/docker-compose.yml +++ b/external-import/phishunt/docker-compose.yml @@ -12,5 +12,9 @@ services : - CONNECTOR_LOG_LEVEL=error - PHISHUNT_API_KEY= # Optional, if not provided, consume only https://phishunt.io/feed.txt - PHISHUNT_CREATE_INDICATORS=true + - PHISHUNT_DEFAULT_X_OPENCTI_SCORE=40 # Optional: default is 40 + - PHISHUNT_X_OPENCTI_SCORE_DOMAIN=40 # Optional + - PHISHUNT_X_OPENCTI_SCORE_IP=40 # Optional + - PHISHUNT_X_OPENCTI_SCORE_URL=60 # Optional - PHISHUNT_INTERVAL=3 # In days, must be strictly greater than 1 restart: always diff --git a/external-import/phishunt/src/config.yml.sample b/external-import/phishunt/src/config.yml.sample index 6247306072..d582c9e552 100644 --- a/external-import/phishunt/src/config.yml.sample +++ b/external-import/phishunt/src/config.yml.sample @@ -15,4 +15,8 @@ connector: phishunt: api_key: '' # Optional, if not provided, consume only https://phishunt.io/feed.txt create_indicators: True + default_x_opencti_score: 40 # Optional: Defaults to 40 + x_opencti_score_domain: 40 # Optional + x_opencti_score_ip: 40 # Optional + x_opencti_score_url: 60 # Optional interval: 3 # In days, must be strictly greater than 1 diff --git a/external-import/phishunt/src/phishunt.py b/external-import/phishunt/src/phishunt.py index 90c67e846d..14838f0685 100644 --- a/external-import/phishunt/src/phishunt.py +++ b/external-import/phishunt/src/phishunt.py @@ -46,6 +46,38 @@ def __init__(self): False, default=True, ) + self.default_x_opencti_score = get_config_variable( + "PHISHUNT_DEFAULT_X_OPENCTI_SCORE", + ["phishunt", "default_x_opencti_score"], + config, + isNumber=True, + default=40, + required=False, + ) + self.x_opencti_score_domain = get_config_variable( + "PHISHUNT_X_OPENCTI_SCORE_DOMAIN", + ["phishunt", "x_opencti_score_domain"], + config, + isNumber=True, + default=None, + required=False, + ) + self.x_opencti_score_ip = get_config_variable( + "PHISHUNT_X_OPENCTI_SCORE_IP", + ["phishunt", "x_opencti_score_ip"], + config, + isNumber=True, + default=None, + required=False, + ) + self.x_opencti_score_url = get_config_variable( + "PHISHUNT_X_OPENCTI_SCORE_URL", + ["phishunt", "x_opencti_score_url"], + config, + isNumber=True, + default=None, + required=False, + ) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], @@ -94,7 +126,8 @@ def _process_public_feed(self, work_id): object_marking_refs=[stix2.TLP_WHITE], custom_properties={ "x_opencti_description": "Phishunt malicious URL", - "x_opencti_score": 40, + "x_opencti_score": self.x_opencti_score_url + or self.default_x_opencti_score, "x_opencti_labels": ["osint", "phishing"], "x_opencti_created_by_ref": self.identity["standard_id"], }, @@ -135,8 +168,8 @@ def _process_public_feed(self, work_id): ) if os.path.exists(os.path.dirname(os.path.abspath(__file__)) + "/data.txt"): os.remove(os.path.dirname(os.path.abspath(__file__)) + "/data.txt") - except Exception as e: - self.helper.log_error(str(e)) + except Exception as error: + self.helper.log_error(str(error)) def _process_private_feed(self, work_id): try: @@ -153,7 +186,8 @@ def _process_private_feed(self, work_id): object_marking_refs=[stix2.TLP_WHITE], custom_properties={ "x_opencti_description": "Phishunt malicious URL", - "x_opencti_score": 40, + "x_opencti_score": self.x_opencti_score_url + or self.default_x_opencti_score, "x_opencti_labels": ["osint", "phishing"], "x_opencti_created_by_ref": self.identity["standard_id"], }, @@ -191,7 +225,8 @@ def _process_private_feed(self, work_id): object_marking_refs=[stix2.TLP_WHITE], custom_properties={ "x_opencti_description": "Phishunt domain based on malicious URL", - "x_opencti_score": 40, + "x_opencti_score": self.x_opencti_score_domain + or self.default_x_opencti_score, "x_opencti_labels": ["osint", "phishing"], "x_opencti_created_by_ref": self.identity["standard_id"], }, @@ -233,7 +268,8 @@ def _process_private_feed(self, work_id): object_marking_refs=[stix2.TLP_WHITE], custom_properties={ "x_opencti_description": "Phishunt domain based on malicious URL", - "x_opencti_score": 40, + "x_opencti_score": self.x_opencti_score_ip + or self.default_x_opencti_score, "x_opencti_labels": ["osint", "phishing"], "x_opencti_created_by_ref": self.identity["standard_id"], }, @@ -276,8 +312,8 @@ def _process_private_feed(self, work_id): update=self.update_existing_data, work_id=work_id, ) - except Exception as e: - self.helper.log_error(str(e)) + except Exception as error: + self.helper.log_error(str(error)) def run(self): self.helper.log_info("Fetching Phishunt dataset...") @@ -338,8 +374,8 @@ def run(self): except (KeyboardInterrupt, SystemExit): self.helper.log_info("Connector stop") sys.exit(0) - except Exception as e: - self.helper.log_error(str(e)) + except Exception as error: + self.helper.log_error(str(error)) if self.helper.connect_run_and_terminate: self.helper.log_info("Connector stop") diff --git a/external-import/threatfox/README.md b/external-import/threatfox/README.md index d671aeee4a..dda890db8d 100644 --- a/external-import/threatfox/README.md +++ b/external-import/threatfox/README.md @@ -45,5 +45,10 @@ The connector adds the following Entities: | `csv_url` | `THREATFOX_CSV_URL` | No | Defaults to `https://threatfox.abuse.ch/export/csv/recent/` | | `import_offline` | `THREATFOX_IMPORT_OFFLINE` | No | Create records for indicators that are offline. Defaults to `True` | | `create_indicators` | `THREATFOX_CREATE_INDICATORS` | No | Create indicators in addition to observables. Defaults to `True` | -| `interval` | `THREATFOX_INTERVAL` | No | Run interval. Defaults to `3` | +| `default_x_opencti_score` | `THREATFOX_DEFAULT_X_OPENCTI_SCORE` | No | The default x_opencti_score to use. Defaults to `50`. | +| `x_opencti_score_ip` | `THREATFOX_X_OPENCTI_SCORE_IP` | No | Set the x_opencti_score for IP observables/indicators. | +| `x_opencti_score_domain` | `THREATFOX_X_OPENCTI_SCORE_DOMAIN` | No | Set the x_opencti_score for Domain observables/indicators. | +| `x_opencti_score_url` | `THREATFOX_X_OPENCTI_SCORE_URL` | No | Set the x_opencti_score for URL observables/indicators. | +| `x_opencti_score_hash` | `THREATFOX_X_OPENCTI_SCORE_HASH` | No | Set the x_opencti_score for Hash observables/indicators. | +| `interval` | `THREATFOX_INTERVAL` | No | Run interval. Defaults to `3` | | `ioc_types` | `THREATFOX_IOC_TYPES` | No | List of IOC types to retrieve, available parameter: `all_types, ip:port, domain, url, md5_hash, sha1_hash, sha256_hash` | diff --git a/external-import/threatfox/docker-compose.yml b/external-import/threatfox/docker-compose.yml index c171f79b2e..4f09a9f1b7 100644 --- a/external-import/threatfox/docker-compose.yml +++ b/external-import/threatfox/docker-compose.yml @@ -13,6 +13,11 @@ services: - THREATFOX_CSV_URL=https://threatfox.abuse.ch/export/csv/recent/ - THREATFOX_IMPORT_OFFLINE=true - THREATFOX_CREATE_INDICATORS=true + - THREATFOX_DEFAULT_X_OPENCTI_SCORE=50 + - THREATFOX_X_OPENCTI_SCORE_IP=60 + - THREATFOX_X_OPENCTI_SCORE_DOMAIN=70 + - THREATFOX_X_OPENCTI_SCORE_URL=75 + - THREATFOX_X_OPENCTI_SCORE_HASH=80 - THREATFOX_INTERVAL=3 # In days, must be strictly greater than 1 - THREATFOX_IOC_TO_IMPORT=ip:port,domain,url # List of IOC types to import restart: always diff --git a/external-import/threatfox/src/config.yml.sample b/external-import/threatfox/src/config.yml.sample index 6254f0ab45..2faa95f694 100644 --- a/external-import/threatfox/src/config.yml.sample +++ b/external-import/threatfox/src/config.yml.sample @@ -15,5 +15,10 @@ threatfox: csv_url: 'https://threatfox.abuse.ch/export/csv/full/' import_offline: true create_indicators: true + default_x_opencti_score: 50 # Optional: Default is 50 + x_opencti_score_ip: 60 # Optional + x_opencti_score_domain: 70 # Optional + x_opencti_score_url: 75 # Optional + x_opencti_score_hash: 80 # Optional interval: 3 # In days, must be strictly greater than 1 ioc_to_import:'ip:port,domain,url' # List of IOC types to import \ No newline at end of file diff --git a/external-import/threatfox/src/main.py b/external-import/threatfox/src/main.py index 714f57d493..37b95ea383 100644 --- a/external-import/threatfox/src/main.py +++ b/external-import/threatfox/src/main.py @@ -74,6 +74,46 @@ def __init__(self): config, default=True, ) + self.default_x_opencti_score: int = get_config_variable( + "THREATFOX_DEFAULT_X_OPENCTI_SCORE", + ["threatfox", "default_x_opencti_score"], + config, + isNumber=True, + default=50, + required=False, + ) + self.x_opencti_score_ip: int = get_config_variable( + "THREATFOX_X_OPENCTI_SCORE_IP", + ["threatfox", "x_opencti_score_ip"], + config, + isNumber=True, + default=None, + required=False, + ) + self.x_opencti_score_domain: int = get_config_variable( + "THREATFOX_X_OPENCTI_SCORE_DOMAIN", + ["threatfox", "x_opencti_score_domain"], + config, + isNumber=True, + default=None, + required=False, + ) + self.x_opencti_score_url: int = get_config_variable( + "THREATFOX_X_OPENCTI_SCORE_URL", + ["threatfox", "x_opencti_score_url"], + config, + isNumber=True, + default=None, + required=False, + ) + self.x_opencti_score_hash: int = get_config_variable( + "THREATFOX_X_OPENCTI_SCORE_HASH", + ["threatfox", "x_opencti_score_hash"], + config, + isNumber=True, + default=None, + required=False, + ) self.ioc_to_import: list[str] = get_config_variable( "THREATFOX_IOC_TO_IMPORT", ["threatfox", "ioc_to_import"], @@ -104,6 +144,19 @@ def get_interval(self) -> float: return float(self.threatfox_interval) * 60 * 60 * 24 + def get_x_opencti_score(self, observable_type: str) -> int: + """Get the x_opencti_score for the observable type""" + + if observable_type == "IPv4-Addr": + return self.x_opencti_score_ip or self.default_x_opencti_score + if observable_type == "Domain-Name": + return self.x_opencti_score_domain or self.default_x_opencti_score + if observable_type == "Url": + return self.x_opencti_score_url or self.default_x_opencti_score + if observable_type == "StixFile": + return self.x_opencti_score_hash or self.default_x_opencti_score + return self.default_x_opencti_score + def run(self): """Run the connector""" @@ -316,6 +369,9 @@ def process_row_observable( "created_by_ref": self.identity["standard_id"], "x_opencti_description": description, "x_opencti_labels": ioc.tags, + "x_opencti_score": ( + self.x_opencti_score_ip or self.default_x_opencti_score + ), }, ) elif ioc.type == "domain": @@ -329,6 +385,9 @@ def process_row_observable( "created_by_ref": self.identity["standard_id"], "x_opencti_description": description, "x_opencti_labels": ioc.tags, + "x_opencti_score": ( + self.x_opencti_score_domain or self.default_x_opencti_score + ), }, ) elif ioc.type == "url": @@ -342,6 +401,9 @@ def process_row_observable( "created_by_ref": self.identity["standard_id"], "x_opencti_description": description, "x_opencti_labels": ioc.tags, + "x_opencti_score": ( + self.x_opencti_score_url or self.default_x_opencti_score + ), }, ) elif ioc.type == "md5_hash": @@ -356,6 +418,9 @@ def process_row_observable( "created_by_ref": self.identity["standard_id"], "x_opencti_description": description, "x_opencti_labels": ioc.tags, + "x_opencti_score": ( + self.x_opencti_score_hash or self.default_x_opencti_score + ), }, ) elif ioc.type == "sha1_hash": @@ -370,6 +435,9 @@ def process_row_observable( "created_by_ref": self.identity["standard_id"], "x_opencti_description": description, "x_opencti_labels": ioc.tags, + "x_opencti_score": ( + self.x_opencti_score_hash or self.default_x_opencti_score + ), }, ) elif ioc.type == "sha256_hash": @@ -384,6 +452,9 @@ def process_row_observable( "created_by_ref": self.identity["standard_id"], "x_opencti_description": description, "x_opencti_labels": ioc.tags, + "x_opencti_score": ( + self.x_opencti_score_hash or self.default_x_opencti_score + ), }, ) else: @@ -417,6 +488,7 @@ def process_row_observable( external_references=ext_refs, custom_properties={ "x_opencti_main_observable_type": observable_type, + "x_opencti_score": self.get_x_opencti_score(observable_type), }, ) self.helper.log_debug(f"Indicator created: {stix_indicator}") diff --git a/external-import/urlhaus/docker-compose.yml b/external-import/urlhaus/docker-compose.yml index 41f2f0bc8c..aabe1beedb 100644 --- a/external-import/urlhaus/docker-compose.yml +++ b/external-import/urlhaus/docker-compose.yml @@ -11,6 +11,7 @@ services: - CONNECTOR_CONFIDENCE_LEVEL=40 # From 0 (Unknown) to 100 (Fully trusted) - CONNECTOR_LOG_LEVEL=error - URLHAUS_CSV_URL=https://urlhaus.abuse.ch/downloads/csv_recent/ + - URLHAUS_DEFAULT_X_OPENCTI_SCORE=80 # Optional: Defaults to 80. - URLHAUS_IMPORT_OFFLINE=true - URLHAUS_THREATS_FROM_LABELS=true - URLHAUS_INTERVAL=3 # In days, must be strictly greater than 1 diff --git a/external-import/urlhaus/src/config.yml.sample b/external-import/urlhaus/src/config.yml.sample index 9e39278e09..408fa0a30f 100644 --- a/external-import/urlhaus/src/config.yml.sample +++ b/external-import/urlhaus/src/config.yml.sample @@ -13,6 +13,7 @@ connector: urlhaus: csv_url: 'https://urlhaus.abuse.ch/downloads/csv_recent/' + default_x_opencti_score: 80 # Optional: Defaults to 80 import_offline: true threats_from_labels: true interval: 3 # In days, must be strictly greater than 1 \ No newline at end of file diff --git a/external-import/urlhaus/src/urlhaus.py b/external-import/urlhaus/src/urlhaus.py index 6b239a49f1..c812b584f8 100644 --- a/external-import/urlhaus/src/urlhaus.py +++ b/external-import/urlhaus/src/urlhaus.py @@ -45,6 +45,14 @@ def __init__(self): False, True, ) + self.default_x_opencti_score = get_config_variable( + "URLHAUS_DEFAULT_X_OPENCTI_SCORE", + ["urlhaus", "default_x_opencti_score"], + config=config, + isNumber=True, + default=80, + required=False, + ) self.update_existing_data = get_config_variable( "CONNECTOR_UPDATE_EXISTING_DATA", ["connector", "update_existing_data"], @@ -175,7 +183,7 @@ def run(self): external_references=[external_reference], object_marking_refs=[stix2.TLP_WHITE], custom_properties={ - "x_opencti_score": 80, + "x_opencti_score": self.default_x_opencti_score, "x_opencti_main_observable_type": "Url", }, ) @@ -189,7 +197,7 @@ def run(self): + row[8] + " - Status: " + row[3], - "x_opencti_score": 80, + "x_opencti_score": self.default_x_opencti_score, "labels": [x for x in row[6].split(",") if x], "created_by_ref": self.identity["standard_id"], "external_references": [external_reference],