diff --git a/internal-enrichment/hygiene/src/hygiene.py b/internal-enrichment/hygiene/src/hygiene.py index 9a99d849d8..a4c3eb21e8 100644 --- a/internal-enrichment/hygiene/src/hygiene.py +++ b/internal-enrichment/hygiene/src/hygiene.py @@ -17,86 +17,94 @@ # Reference: https://github.com/MISP/misp-warninglists/issues/142 # To generate: grep '"name"' -r lists, and then reformat using vscode LIST_MAPPING = { - "List of known Apple IP ranges": "lists/apple/list.json", - "List of known SMTP receiving IP addresses": "lists/smtp-receiving-ips/list.json", - "List of known Gmail sending IP ranges": "lists/google-gmail-sending-ips/list.json", - "List of known domains to know external IP": "lists/whats-my-ip/list.json", - "Top 500 domains and pages from https://moz.com/top500": "lists/moz-top500/list.json", - "List of known Windows 10 connection endpoints": "lists/microsoft-win10-connection-endpoints/list.json", - "List of known security providers/vendors blog domain": "lists/security-provider-blogpost/list.json", - "List of known hashes with common false-positives (based on Florian Roth input list)": "lists/common-ioc-false-positive/list.json", - "Top 20 000 websites from Cisco Umbrella": "lists/cisco_top20k/list.json", - "Specialized list of vpn-ipv4 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv4/list.json", - "List of known SMTP sending IP ranges": "lists/smtp-sending-ips/list.json", - "List of known Office 365 IP address ranges in China": "lists/microsoft-office365-cn/list.json", - "List of RFC 5735 CIDR blocks": "lists/rfc5735/list.json", - "List of RFC 5771 multicast CIDR blocks": "lists/multicast/list.json", - "List of known Microsoft Azure US Government Cloud Datacenter IP Ranges": "lists/microsoft-azure-us-gov/list.json", - "List of known GCP (Google Cloud Platform) IP address ranges": "lists/google-gcp/list.json", - "List of RFC 1918 CIDR blocks": "lists/rfc1918/list.json", + "List of known Akamai IP ranges": "lists/akamai/list.json", "Top 1000 website from Alexa": "lists/alexa/list.json", - "CRL and OCSP domains": "lists/crl-hostname/list.json", - "List of known Office 365 URLs": "lists/microsoft-office365/list.json", - "Hashes that are often included in IOC lists but are false positives.": "lists/ti-falsepositives/list.json", - "List of known bank domains": "lists/bank-website/list.json", - "List of known IPv6 public DNS resolvers": "lists/public-dns-v6/list.json", - "List of known google domains": "lists/google/list.json", - "List of known microsoft domains": "lists/microsoft/list.json", - "Parking domains name server": "lists/parking-domain-ns/list.json", - "List of known Tenable Cloud Sensors IPv6": "lists/tenable-cloud-ipv6/list.json", - "List of known Ovh Cluster IP": "lists/ovh-cluster/list.json", + "List of known Amazon AWS IP address ranges": "lists/amazon-aws/list.json", + "List of known Apple IP ranges": "lists/apple/list.json", "List of known domains used by automated malware analysis services & security vendors": "lists/automated-malware-analysis/list.json", - "List of known Cloudflare IP ranges": "lists/cloudflare/list.json", + "List of known bank domains": "lists/bank-website/list.json", + "Captive Portal Detection Hostnames": "lists/captive-portals/list.json", + "Censys IP Ranges Used for Scanning": "lists/censys-scanning/list.json", + "List of known check-host.net IP address ranges": "lists/check-host-net/list.json", + "Top 1000 websites from Cisco Umbrella": "lists/cisco_top1000/list.json", "Top 10 000 websites from Cisco Umbrella": "lists/cisco_top10k/list.json", - "google-chrome-crux-1million": "lists/google-chrome-crux-1million/list.json", - "List of known hashes for empty files": "lists/empty-hashes/list.json", - "List of known Fastly IP address ranges": "lists/fastly/list.json", + "Top 20 000 websites from Cisco Umbrella": "lists/cisco_top20k/list.json", + "Top 5000 websites from Cisco Umbrella": "lists/cisco_top5k/list.json", + "List of known Cloudflare IP ranges": "lists/cloudflare/list.json", "Common contact e-mail addresses": "lists/common-contact-emails/list.json", - "Fingerprint of trusted CA certificates": "lists/mozilla-CA/list.json", - "Captive Portal Detection Hostnames": "lists/captive-portals/list.json", + "List of known hashes with common false-positives (based on Florian Roth input list)": "lists/common-ioc-false-positive/list.json", + "Valid covid-19 related domains": "lists/covid/list.json", "Covid-19 Cyber Threat Coalition's Whitelist": "lists/covid-19-cyber-threat-coalition-whitelist/list.json", - "List of known Akamai IP ranges": "lists/akamai/list.json", - "Specialized list of IPv6 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv6/list.json", - "List of known Microsoft Azure Datacenter IP Ranges": "lists/microsoft-azure/list.json", - "List of known public IPFS gateways": "lists/public-ipfs-gateways/list.json", - "List of IPv6 link local blocks": "lists/ipv6-linklocal/list.json", - "List of known public DNS resolvers expressed as hostname": "lists/public-dns-hostname/list.json", - "Parking domains": "lists/parking-domain/list.json", - "List of known hashes for benign files": "lists/nioc-filehash/list.json", - "Top 1000 websites from Cisco Umbrella": "lists/cisco_top1000/list.json", - "List of known Stackpath CDN IP ranges": "lists/stackpath/list.json", + "Covid-19 Krassi's Whitelist": "lists/covid-19-krassi-whitelist/list.json", + "CRL and OCSP domains": "lists/crl-hostname/list.json", + "CRL and OCSP IP addresses": "lists/crl-ip/list.json", + "List of known dax30 webpages": "lists/dax30/list.json", + "OSINT.DigitalSide.IT Warning List": "lists/digitalside/list.json", + "List of disposable email domains": "lists/disposable-email/list.json", + "List of known dynamic DNS domains": "lists/dynamic-dns/list.json", "List of hashes for EICAR test virus": "lists/eicar.com/list.json", - "University domains": "lists/university_domains/list.json", - "List of known Office 365 IP address ranges": "lists/microsoft-office365-ip/list.json", - "Top 10K most-used sites from Tranco": "lists/tranco10k/list.json", - "List of known Amazon AWS IP address ranges": "lists/amazon-aws/list.json", + "List of known hashes for empty files": "lists/empty-hashes/list.json", + "List of known Fastly IP address ranges": "lists/fastly/list.json", + "List of known hostname used for querying your source IP. This can be used as exclusion for your Passive DNS lookup.": "lists/findip-host/list.json", + "List of known google domains": "lists/google/list.json", + "google-chrome-crux-1million": "lists/google-chrome-crux-1million/list.json", + "List of known GCP (Google Cloud Platform) IP address ranges": "lists/google-gcp/list.json", + "List of known Gmail sending IP ranges": "lists/google-gmail-sending-ips/list.json", "List of known Googlebot IP ranges (https://developers.google.com/search/apis/ipranges/googlebot.json)": "lists/googlebot/list.json", "TLDs as known by IANA": "lists/tlds/list.json", - "Top 5000 websites from Cisco Umbrella": "lists/cisco_top5k/list.json", + "List of IPv6 link local blocks": "lists/ipv6-linklocal/list.json", + "List of known link in Bio domains": "lists/link-in-bio/list.json", + "Top 10000 websites from Majestic Million": "lists/majestic_million/list.json", + "List of known microsoft domains": "lists/microsoft/list.json", + "List of known Office 365 Attack Simulator used for phishing awareness campaigns": "lists/microsoft-attack-simulator/list.json", + "List of known Microsoft Azure Datacenter IP Ranges": "lists/microsoft-azure/list.json", + "List of Azure Applicaiton IDs": "lists/microsoft-azure-appid/list.json", + "List of known Microsoft Azure China Datacenter IP Ranges": "lists/microsoft-azure-china/list.json", + "List of known Microsoft Azure Germany Datacenter IP Ranges": "lists/microsoft-azure-germany/list.json", + "List of known Microsoft Azure US Government Cloud Datacenter IP Ranges": "lists/microsoft-azure-us-gov/list.json", + "List of known Office 365 URLs": "lists/microsoft-office365/list.json", + "List of known Office 365 IP address ranges in China": "lists/microsoft-office365-cn/list.json", + "List of known Office 365 IP address ranges": "lists/microsoft-office365-ip/list.json", + "List of known Windows 10 connection endpoints": "lists/microsoft-win10-connection-endpoints/list.json", + "Top 500 domains and pages from https://moz.com/top500": "lists/moz-top500/list.json", + "Fingerprint of trusted CA certificates": "lists/mozilla-CA/list.json", + "Fingerprint of known intermediate of trusted certificates": "lists/mozilla-IntermediateCA/list.json", + "List of RFC 5771 multicast CIDR blocks": "lists/multicast/list.json", + "List of known hashes for benign files": "lists/nioc-filehash/list.json", + "List of known IP address ranges for OpenAI GPT crawler bot": "lists/openai-gptbot/list.json", + "List of known Ovh Cluster IP": "lists/ovh-cluster/list.json", + "Parking domains": "lists/parking-domain/list.json", + "Parking domains name server": "lists/parking-domain-ns/list.json", "Unattributed phone number.": "lists/phone_numbers/list.json", + "List of known public DNS resolvers expressed as hostname": "lists/public-dns-hostname/list.json", + "List of known IPv4 public DNS resolvers": "lists/public-dns-v4/list.json", + "List of known IPv6 public DNS resolvers": "lists/public-dns-v6/list.json", + "List of known public IPFS gateways": "lists/public-ipfs-gateways/list.json", + "List of RFC 1918 CIDR blocks": "lists/rfc1918/list.json", "List of RFC 3849 CIDR blocks": "lists/rfc3849/list.json", - "List of known Office 365 Attack Simulator used for phishing awareness campaigns": "lists/microsoft-attack-simulator/list.json", - "List of RFC 6761 Special-Use Domain Names": "lists/rfc6761/list.json", + "List of RFC 5735 CIDR blocks": "lists/rfc5735/list.json", "List of RFC 6598 CIDR blocks": "lists/rfc6598/list.json", + "List of RFC 6761 Special-Use Domain Names": "lists/rfc6761/list.json", + "List of known security providers/vendors blog domain": "lists/security-provider-blogpost/list.json", + "List of known sinkholes": "lists/sinkholes/list.json", + "List of known SMTP receiving IP addresses": "lists/smtp-receiving-ips/list.json", + "List of known SMTP sending IP ranges": "lists/smtp-sending-ips/list.json", + "List of known Stackpath CDN IP ranges": "lists/stackpath/list.json", "List of known Tenable Cloud Sensors IPv4": "lists/tenable-cloud-ipv4/list.json", - "List of known IPv4 public DNS resolvers": "lists/public-dns-v4/list.json", - "List of known dax30 webpages": "lists/dax30/list.json", - "List of disposable email domains": "lists/disposable-email/list.json", - "Top 1,000,000 most-used sites from Tranco": "lists/tranco/list.json", - "List of known Microsoft Azure Germany Datacenter IP Ranges": "lists/microsoft-azure-germany/list.json", - "Valid covid-19 related domains": "lists/covid/list.json", - "List of known dynamic DNS domains": "lists/dynamic-dns/list.json", - "Top 10000 websites from Majestic Million": "lists/majestic_million/list.json", - "CRL and OCSP IP addresses": "lists/crl-ip/list.json", + "List of known Tenable Cloud Sensors IPv6": "lists/tenable-cloud-ipv6/list.json", + "Hashes that are often included in IOC lists but are false positives.": "lists/ti-falsepositives/list.json", + "Top 1000000 most-used sites from Tranco": "lists/tranco/list.json", + "Top 10K most-used sites from Tranco": "lists/tranco10k/list.json", + "cisco-umbrella-blockpage-hostname": "lists/umbrella-blockpage-hostname/list.json", + "cisco-umbrella-blockpage-ipv4": "lists/umbrella-blockpage-v4/list.json", + "cisco-umbrella-blockpage-ipv6": "lists/umbrella-blockpage-v6/list.json", + "University domains": "lists/university_domains/list.json", "List of known URL Shorteners domains": "lists/url-shortener/list.json", - "Covid-19 Krassi's Whitelist": "lists/covid-19-krassi-whitelist/list.json", + "Specialized list of vpn-ipv4 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv4/list.json", + "Specialized list of IPv6 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv6/list.json", + "List of known domains to know external IP": "lists/whats-my-ip/list.json", "List of known Wikimedia address ranges": "lists/wikimedia/list.json", - "List of known sinkholes": "lists/sinkholes/list.json", - "List of known Microsoft Azure China Datacenter IP Ranges": "lists/microsoft-azure-china/list.json", - "Second level TLDs as known by Mozilla Foundation": "lists/second-level-tlds/list.json", - "List of Azure Applicaiton IDs": "lists/microsoft-azure-appid/list.json", - "Fingerprint of known intermediate of trusted certificates": "lists/mozilla-IntermediateCA/list.json", - "List of known hostname used for querying your source IP. This can be used as exclusion for your Passive DNS lookup.": "lists/findip-host/list.json", + "List of known Zscaler IP address ranges": "lists/zscaler/list.json", }