Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update MISP lists in hygiene connector #2839

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Update MISP lists in hygiene connector #2839

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d77546
[Add] More Lists to hygiene
baptiste-fourmont Oct 24, 2024
a17753e
[Add] Unattributed phone number
baptiste-fourmont Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 74 additions & 66 deletions internal-enrichment/hygiene/src/hygiene.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,86 +17,94 @@
# Reference: https://github.com/MISP/misp-warninglists/issues/142
# To generate: grep '"name"' -r lists, and then reformat using vscode
LIST_MAPPING = {
"List of known Apple IP ranges": "lists/apple/list.json",
"List of known SMTP receiving IP addresses": "lists/smtp-receiving-ips/list.json",
"List of known Gmail sending IP ranges": "lists/google-gmail-sending-ips/list.json",
"List of known domains to know external IP": "lists/whats-my-ip/list.json",
"Top 500 domains and pages from https://moz.com/top500": "lists/moz-top500/list.json",
"List of known Windows 10 connection endpoints": "lists/microsoft-win10-connection-endpoints/list.json",
"List of known security providers/vendors blog domain": "lists/security-provider-blogpost/list.json",
"List of known hashes with common false-positives (based on Florian Roth input list)": "lists/common-ioc-false-positive/list.json",
"Top 20 000 websites from Cisco Umbrella": "lists/cisco_top20k/list.json",
"Specialized list of vpn-ipv4 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv4/list.json",
"List of known SMTP sending IP ranges": "lists/smtp-sending-ips/list.json",
"List of known Office 365 IP address ranges in China": "lists/microsoft-office365-cn/list.json",
"List of RFC 5735 CIDR blocks": "lists/rfc5735/list.json",
"List of RFC 5771 multicast CIDR blocks": "lists/multicast/list.json",
"List of known Microsoft Azure US Government Cloud Datacenter IP Ranges": "lists/microsoft-azure-us-gov/list.json",
"List of known GCP (Google Cloud Platform) IP address ranges": "lists/google-gcp/list.json",
"List of RFC 1918 CIDR blocks": "lists/rfc1918/list.json",
"List of known Akamai IP ranges": "lists/akamai/list.json",
"Top 1000 website from Alexa": "lists/alexa/list.json",
"CRL and OCSP domains": "lists/crl-hostname/list.json",
"List of known Office 365 URLs": "lists/microsoft-office365/list.json",
"Hashes that are often included in IOC lists but are false positives.": "lists/ti-falsepositives/list.json",
"List of known bank domains": "lists/bank-website/list.json",
"List of known IPv6 public DNS resolvers": "lists/public-dns-v6/list.json",
"List of known google domains": "lists/google/list.json",
"List of known microsoft domains": "lists/microsoft/list.json",
"Parking domains name server": "lists/parking-domain-ns/list.json",
"List of known Tenable Cloud Sensors IPv6": "lists/tenable-cloud-ipv6/list.json",
"List of known Ovh Cluster IP": "lists/ovh-cluster/list.json",
"List of known Amazon AWS IP address ranges": "lists/amazon-aws/list.json",
"List of known Apple IP ranges": "lists/apple/list.json",
"List of known domains used by automated malware analysis services & security vendors": "lists/automated-malware-analysis/list.json",
"List of known Cloudflare IP ranges": "lists/cloudflare/list.json",
"List of known bank domains": "lists/bank-website/list.json",
"Captive Portal Detection Hostnames": "lists/captive-portals/list.json",
"Censys IP Ranges Used for Scanning": "lists/censys-scanning/list.json",
"List of known check-host.net IP address ranges": "lists/check-host-net/list.json",
"Top 1000 websites from Cisco Umbrella": "lists/cisco_top1000/list.json",
"Top 10 000 websites from Cisco Umbrella": "lists/cisco_top10k/list.json",
"google-chrome-crux-1million": "lists/google-chrome-crux-1million/list.json",
"List of known hashes for empty files": "lists/empty-hashes/list.json",
"List of known Fastly IP address ranges": "lists/fastly/list.json",
"Top 20 000 websites from Cisco Umbrella": "lists/cisco_top20k/list.json",
"Top 5000 websites from Cisco Umbrella": "lists/cisco_top5k/list.json",
"List of known Cloudflare IP ranges": "lists/cloudflare/list.json",
"Common contact e-mail addresses": "lists/common-contact-emails/list.json",
"Fingerprint of trusted CA certificates": "lists/mozilla-CA/list.json",
"Captive Portal Detection Hostnames": "lists/captive-portals/list.json",
"List of known hashes with common false-positives (based on Florian Roth input list)": "lists/common-ioc-false-positive/list.json",
"Valid covid-19 related domains": "lists/covid/list.json",
"Covid-19 Cyber Threat Coalition's Whitelist": "lists/covid-19-cyber-threat-coalition-whitelist/list.json",
"List of known Akamai IP ranges": "lists/akamai/list.json",
"Specialized list of IPv6 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv6/list.json",
"List of known Microsoft Azure Datacenter IP Ranges": "lists/microsoft-azure/list.json",
"List of known public IPFS gateways": "lists/public-ipfs-gateways/list.json",
"List of IPv6 link local blocks": "lists/ipv6-linklocal/list.json",
"List of known public DNS resolvers expressed as hostname": "lists/public-dns-hostname/list.json",
"Parking domains": "lists/parking-domain/list.json",
"List of known hashes for benign files": "lists/nioc-filehash/list.json",
"Top 1000 websites from Cisco Umbrella": "lists/cisco_top1000/list.json",
"List of known Stackpath CDN IP ranges": "lists/stackpath/list.json",
"Covid-19 Krassi's Whitelist": "lists/covid-19-krassi-whitelist/list.json",
"CRL and OCSP domains": "lists/crl-hostname/list.json",
"CRL and OCSP IP addresses": "lists/crl-ip/list.json",
"List of known dax30 webpages": "lists/dax30/list.json",
"OSINT.DigitalSide.IT Warning List": "lists/digitalside/list.json",
"List of disposable email domains": "lists/disposable-email/list.json",
"List of known dynamic DNS domains": "lists/dynamic-dns/list.json",
"List of hashes for EICAR test virus": "lists/eicar.com/list.json",
"University domains": "lists/university_domains/list.json",
"List of known Office 365 IP address ranges": "lists/microsoft-office365-ip/list.json",
"Top 10K most-used sites from Tranco": "lists/tranco10k/list.json",
"List of known Amazon AWS IP address ranges": "lists/amazon-aws/list.json",
"List of known hashes for empty files": "lists/empty-hashes/list.json",
"List of known Fastly IP address ranges": "lists/fastly/list.json",
"List of known hostname used for querying your source IP. This can be used as exclusion for your Passive DNS lookup.": "lists/findip-host/list.json",
"List of known google domains": "lists/google/list.json",
"google-chrome-crux-1million": "lists/google-chrome-crux-1million/list.json",
"List of known GCP (Google Cloud Platform) IP address ranges": "lists/google-gcp/list.json",
"List of known Gmail sending IP ranges": "lists/google-gmail-sending-ips/list.json",
"List of known Googlebot IP ranges (https://developers.google.com/search/apis/ipranges/googlebot.json)": "lists/googlebot/list.json",
"TLDs as known by IANA": "lists/tlds/list.json",
"Top 5000 websites from Cisco Umbrella": "lists/cisco_top5k/list.json",
"List of IPv6 link local blocks": "lists/ipv6-linklocal/list.json",
"List of known link in Bio domains": "lists/link-in-bio/list.json",
"Top 10000 websites from Majestic Million": "lists/majestic_million/list.json",
"List of known microsoft domains": "lists/microsoft/list.json",
"List of known Office 365 Attack Simulator used for phishing awareness campaigns": "lists/microsoft-attack-simulator/list.json",
"List of known Microsoft Azure Datacenter IP Ranges": "lists/microsoft-azure/list.json",
"List of Azure Applicaiton IDs": "lists/microsoft-azure-appid/list.json",
"List of known Microsoft Azure China Datacenter IP Ranges": "lists/microsoft-azure-china/list.json",
"List of known Microsoft Azure Germany Datacenter IP Ranges": "lists/microsoft-azure-germany/list.json",
"List of known Microsoft Azure US Government Cloud Datacenter IP Ranges": "lists/microsoft-azure-us-gov/list.json",
"List of known Office 365 URLs": "lists/microsoft-office365/list.json",
"List of known Office 365 IP address ranges in China": "lists/microsoft-office365-cn/list.json",
"List of known Office 365 IP address ranges": "lists/microsoft-office365-ip/list.json",
"List of known Windows 10 connection endpoints": "lists/microsoft-win10-connection-endpoints/list.json",
"Top 500 domains and pages from https://moz.com/top500": "lists/moz-top500/list.json",
"Fingerprint of trusted CA certificates": "lists/mozilla-CA/list.json",
"Fingerprint of known intermediate of trusted certificates": "lists/mozilla-IntermediateCA/list.json",
"List of RFC 5771 multicast CIDR blocks": "lists/multicast/list.json",
"List of known hashes for benign files": "lists/nioc-filehash/list.json",
"List of known IP address ranges for OpenAI GPT crawler bot": "lists/openai-gptbot/list.json",
"List of known Ovh Cluster IP": "lists/ovh-cluster/list.json",
"Parking domains": "lists/parking-domain/list.json",
"Parking domains name server": "lists/parking-domain-ns/list.json",
"Unattributed phone number.": "lists/phone_numbers/list.json",
"List of known public DNS resolvers expressed as hostname": "lists/public-dns-hostname/list.json",
"List of known IPv4 public DNS resolvers": "lists/public-dns-v4/list.json",
"List of known IPv6 public DNS resolvers": "lists/public-dns-v6/list.json",
"List of known public IPFS gateways": "lists/public-ipfs-gateways/list.json",
"List of RFC 1918 CIDR blocks": "lists/rfc1918/list.json",
"List of RFC 3849 CIDR blocks": "lists/rfc3849/list.json",
"List of known Office 365 Attack Simulator used for phishing awareness campaigns": "lists/microsoft-attack-simulator/list.json",
"List of RFC 6761 Special-Use Domain Names": "lists/rfc6761/list.json",
"List of RFC 5735 CIDR blocks": "lists/rfc5735/list.json",
"List of RFC 6598 CIDR blocks": "lists/rfc6598/list.json",
"List of RFC 6761 Special-Use Domain Names": "lists/rfc6761/list.json",
"List of known security providers/vendors blog domain": "lists/security-provider-blogpost/list.json",
"List of known sinkholes": "lists/sinkholes/list.json",
"List of known SMTP receiving IP addresses": "lists/smtp-receiving-ips/list.json",
"List of known SMTP sending IP ranges": "lists/smtp-sending-ips/list.json",
"List of known Stackpath CDN IP ranges": "lists/stackpath/list.json",
"List of known Tenable Cloud Sensors IPv4": "lists/tenable-cloud-ipv4/list.json",
"List of known IPv4 public DNS resolvers": "lists/public-dns-v4/list.json",
"List of known dax30 webpages": "lists/dax30/list.json",
"List of disposable email domains": "lists/disposable-email/list.json",
"Top 1,000,000 most-used sites from Tranco": "lists/tranco/list.json",
"List of known Microsoft Azure Germany Datacenter IP Ranges": "lists/microsoft-azure-germany/list.json",
"Valid covid-19 related domains": "lists/covid/list.json",
"List of known dynamic DNS domains": "lists/dynamic-dns/list.json",
"Top 10000 websites from Majestic Million": "lists/majestic_million/list.json",
"CRL and OCSP IP addresses": "lists/crl-ip/list.json",
"List of known Tenable Cloud Sensors IPv6": "lists/tenable-cloud-ipv6/list.json",
"Hashes that are often included in IOC lists but are false positives.": "lists/ti-falsepositives/list.json",
"Top 1000000 most-used sites from Tranco": "lists/tranco/list.json",
"Top 10K most-used sites from Tranco": "lists/tranco10k/list.json",
"cisco-umbrella-blockpage-hostname": "lists/umbrella-blockpage-hostname/list.json",
"cisco-umbrella-blockpage-ipv4": "lists/umbrella-blockpage-v4/list.json",
"cisco-umbrella-blockpage-ipv6": "lists/umbrella-blockpage-v6/list.json",
"University domains": "lists/university_domains/list.json",
"List of known URL Shorteners domains": "lists/url-shortener/list.json",
"Covid-19 Krassi's Whitelist": "lists/covid-19-krassi-whitelist/list.json",
"Specialized list of vpn-ipv4 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv4/list.json",
"Specialized list of IPv6 addresses belonging to common VPN providers and datacenters": "lists/vpn-ipv6/list.json",
"List of known domains to know external IP": "lists/whats-my-ip/list.json",
"List of known Wikimedia address ranges": "lists/wikimedia/list.json",
"List of known sinkholes": "lists/sinkholes/list.json",
"List of known Microsoft Azure China Datacenter IP Ranges": "lists/microsoft-azure-china/list.json",
"Second level TLDs as known by Mozilla Foundation": "lists/second-level-tlds/list.json",
"List of Azure Applicaiton IDs": "lists/microsoft-azure-appid/list.json",
"Fingerprint of known intermediate of trusted certificates": "lists/mozilla-IntermediateCA/list.json",
"List of known hostname used for querying your source IP. This can be used as exclusion for your Passive DNS lookup.": "lists/findip-host/list.json",
"List of known Zscaler IP address ranges": "lists/zscaler/list.json",
}


Expand Down